diff --git a/.agents/skills/verify-lane-schedule/model/fingerprints.txt b/.agents/skills/verify-lane-schedule/model/fingerprints.txt index 42532cfbb..f9cdaaa6b 100644 --- a/.agents/skills/verify-lane-schedule/model/fingerprints.txt +++ b/.agents/skills/verify-lane-schedule/model/fingerprints.txt @@ -318,7 +318,7 @@ internal/githttp/hydrate.go:holdFileIdentity 4451039fff307c74d3a416e209140826edc internal/githttp/hydrate.go:holdLineCovers 68e0cad1251be7349a4179f64a25ab38c92ac2e473d6e986c078acaf556520fe internal/githttp/hydrate.go:holdSidecarOwner d344d481c0a6961091973246ecd86c36a1adf26024a7f84daade1a3beb8b72ec internal/githttp/hydrate.go:hydrateOnePack 65e2d6f356099992e8b9e49e8f22855fad64e053f8d4936e87b6ec14b4f382fa -internal/githttp/hydrate.go:hydratePacksBy c8410824050b8b8c2048395646cb00e2a04625bbfead2a0d30b6e0f819e837ca +internal/githttp/hydrate.go:hydratePacksBy 596858a2e0363c40e36ec6db3ef581324457041c17c5f9c42c863d33d88cc624 internal/githttp/hydrate.go:hydrateRefs 10c0aee5ef4b4352de3f3e1d03e899a86951f88b76538a038b6be16f3af840d3 internal/githttp/hydrate.go:import_"bufio" c8716a546ce61a4869563c666058adbec5f4783581ea4f74788f87ec1f0731f1 internal/githttp/hydrate.go:import_"bytes" 151fc787b0679fc8767e367d1971b9f907660a3d72d8fa3888877ef9f8ebd9c3 @@ -421,6 +421,8 @@ internal/githttp/hydrate.go:var_errConfigValueMismatch 6072c7d12edf5f49796836c2e internal/githttp/hydrate.go:var_errPushScanBudget 21c09212ded082bfd4aaab0be40dfe2db6fb09bb6483778ea83d5d7112ff94be internal/githttp/hydrate.go:var_gitSpawnObserver dad15a1d668f4997e06a2c42ecf9814ef4d288463c954ed8503701b311c4d761 internal/githttp/hydrate.go:var_hookVerifyBarrier dec9fba51ee077b370309f5ed4434c9a35961db2b11d69d72854955ca34c4be7 +internal/githttp/hydrate.go:var_hydrateMissingWait 5e73ece8bff435c5d7eaa025de4a7845dd4c89afe3f7f4e23174fcb9569bae36 +internal/githttp/hydrate.go:var_hydrateMissingWaitMax f0661515a42c0fbfa62d8057e46687d5b20c3606976a313e31a9734781e3cd5e internal/githttp/hydrate.go:var_linkFile 47e0a13d524db684a6284b85a674578d5522836e5d22acb6b89909a1d44653d9 internal/githttp/hydrate.go:var_maxPushPeerScanBytes 66203545f4422b2f2c96c6b2ef37e9f1851a855c391a8405cff2b2288909701b internal/githttp/hydrate.go:var_pushHoldTTL d3b2bcf904b9d88d9983388e6f79346bf8b8f4c6466047f7db4f4607ac6b91cb diff --git a/internal/githttp/hydrate.go b/internal/githttp/hydrate.go index 923269120..10bbde88f 100644 --- a/internal/githttp/hydrate.go +++ b/internal/githttp/hydrate.go @@ -886,6 +886,18 @@ func ensureHook(path, script string) (os.FileInfo, error) { // genuinely gone, not racing. const maxHydrateRaces = 8 +// hydrateMissingWait is the first wait between two listings that both find the +// same pack missing (it doubles each time, to hydrateMissingWaitMax). A fold +// that replaced the pack lists the replacement, so a pack that is listed and +// missing twice in a row is no fold race. The orphan sweep can delete the bytes +// of a pack that a writer registered just before, and restores them within one +// pass: a reader that starts inside that window waits for the restore instead +// of failing within milliseconds. Tests shorten it. +var ( + hydrateMissingWait = 50 * time.Millisecond + hydrateMissingWaitMax = time.Second +) + // ErrPackMissing is returned (wrapped, match with errors.Is) when a // registered pack's blob stays missing after maxHydrateRaces listing passes: // the ref store references bytes that do not exist, which is corruption @@ -925,12 +937,15 @@ func hydratePacksBy(ctx context.Context, bl blob.BlobStore, repoPath, repo strin return err } var lastMissing error - for range maxHydrateRaces { + var prevMissingID string + wait := hydrateMissingWait + for attempt := range maxHydrateRaces { packs, err := list(ctx) if err != nil { return err } lastMissing = nil + missingID := "" for _, p := range packs { base := filepath.Join(packDir, "pack-"+p.ID) if _, err := os.Stat(base + ".idx"); err == nil { @@ -941,12 +956,22 @@ func hydratePacksBy(ctx context.Context, bl blob.BlobStore, repoPath, repo strin return err } lastMissing = fmt.Errorf("pack %s: %w", p.ID, err) + missingID = p.ID break } } if lastMissing == nil { return nil } + if missingID == prevMissingID && attempt < maxHydrateRaces-1 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(wait): + } + wait = min(wait*2, hydrateMissingWaitMax) + } + prevMissingID = missingID } return fmt.Errorf("hydrate %s: %w after %d attempts: %v", repo, ErrPackMissing, maxHydrateRaces, lastMissing) } diff --git a/internal/githttp/hydrate_test.go b/internal/githttp/hydrate_test.go index c46e0f659..365f10eac 100644 --- a/internal/githttp/hydrate_test.go +++ b/internal/githttp/hydrate_test.go @@ -679,6 +679,7 @@ func TestHydratePacks_MissingPackExhaustsBound(t *testing.T) { defer bl.Close() gone := ref.Pack{ID: "gone", Key: blob.PackKey(repo, "gone")} st := &scriptedPacks{lists: [][]ref.Pack{{gone}}} + shortenHydrateWait(t) repoPath, err := EnsureCacheRepo(t.TempDir(), repo) if err != nil { @@ -696,6 +697,104 @@ func TestHydratePacks_MissingPackExhaustsBound(t *testing.T) { } } +// shortenHydrateWait makes the wait between two listings that find the same +// pack missing 1 ms for the test. The test must not run in parallel. +func shortenHydrateWait(t *testing.T) { + t.Helper() + oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax + hydrateMissingWait, hydrateMissingWaitMax = time.Millisecond, time.Millisecond + t.Cleanup(func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }) +} + +// timedRestoreStore lists one pack, and starts a timer at the first listing that +// brings the pack's blob back: a sweeper that restores the pack's bytes. +type timedRestoreStore struct { + ref.RefStore + pack ref.Pack + after time.Duration + back func() + calls int +} + +func (s *timedRestoreStore) ListPacks(context.Context, string) ([]ref.Pack, error) { + if s.calls++; s.calls == 1 { + time.AfterFunc(s.after, s.back) + } + return []ref.Pack{s.pack}, nil +} + +// A pack that is listed and missing on consecutive listings is no fold race: +// the sweeper can have deleted the bytes of a pack that a writer registered, and +// restores them within one pass. The hydration waits between the listings for +// the restore. A pack that a fold replaced needs no wait. +func TestHydratePacks_WaitsForAPackThatStaysMissing(t *testing.T) { + ctx := context.Background() + const repo = "t-hydrate-wait" + bl, err := blob.OpenFile("file://" + t.TempDir()) + if err != nil { + t.Fatalf("OpenFile: %v", err) + } + defer bl.Close() + back := ref.Pack{ID: "back", Key: blob.PackKey(repo, "back")} + oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax + hydrateMissingWait, hydrateMissingWaitMax = 100*time.Millisecond, 100*time.Millisecond + defer func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }() + packFile := buildTestPack(t) + repoPath, err := EnsureCacheRepo(t.TempDir(), repo) + if err != nil { + t.Fatalf("EnsureCacheRepo: %v", err) + } + // The blob comes back 250 ms after the first listing: more than the 8 + // listings take without a wait, less than the waits cover. + st := &timedRestoreStore{pack: back, after: 250 * time.Millisecond, back: func() { _ = bl.UploadFile(ctx, back.Key, packFile) }} + if err := HydratePacks(ctx, st, bl, repoPath, repo); err != nil { + t.Fatalf("hydrate of a pack that comes back within the wait: %v", err) + } + if st.calls < 3 { + t.Fatalf("ListPacks called %d times, want at least 3 (the wait covers the first two misses)", st.calls) + } + + // The wait ends with the context. + gone := ref.Pack{ID: "gone", Key: blob.PackKey(repo, "gone")} + hydrateMissingWait, hydrateMissingWaitMax = time.Hour, time.Hour + cctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + start := time.Now() + err = HydratePacks(cctx, &scriptedPacks{lists: [][]ref.Pack{{gone}}}, bl, repoPath, repo) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 5*time.Second { + t.Fatalf("hydrate with a context that ends = %v after %v, want the context error soon", err, time.Since(start)) + } +} + +// A fold race retries at once: the second listing carries the replacement, so +// the missing pack differs and the hydration does not wait. +func TestHydratePacks_FoldRaceRetriesWithoutWaiting(t *testing.T) { + const repo = "t-hydrate-fold-fast" + bl, err := blob.OpenFile("file://" + t.TempDir()) + if err != nil { + t.Fatalf("OpenFile: %v", err) + } + defer bl.Close() + replacement := ref.Pack{ID: "new", Key: blob.PackKey(repo, "new")} + if err := bl.UploadFile(context.Background(), replacement.Key, buildTestPack(t)); err != nil { + t.Fatalf("UploadFile: %v", err) + } + folded := ref.Pack{ID: "old", Key: blob.PackKey(repo, "old")} + oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax + hydrateMissingWait, hydrateMissingWaitMax = time.Hour, time.Hour + defer func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }() + repoPath, err := EnsureCacheRepo(t.TempDir(), repo) + if err != nil { + t.Fatalf("EnsureCacheRepo: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + st := &scriptedPacks{lists: [][]ref.Pack{{folded}, {replacement}}} + if err := HydratePacks(ctx, st, bl, repoPath, repo); err != nil || st.calls != 2 { + t.Fatalf("hydrate across a fold = %v after %d listings, want success after 2 with no wait", err, st.calls) + } +} + // Transport errors must keep failing fast: no re-list, no retry, and no // misclassification as a missing pack. func TestHydratePacks_TransportErrorFailsFast(t *testing.T) { diff --git a/internal/jobs/sweep.go b/internal/jobs/sweep.go index c396acaef..95c125f95 100644 --- a/internal/jobs/sweep.go +++ b/internal/jobs/sweep.go @@ -322,8 +322,11 @@ func NewSweeper(blobs blob.BlobStore, packs PackRegistry, cfg SweepConfig, logge // assumptions (see EnsurePublished's ordering proof). // // Any residual exposure (e.g. a reader racing the restore itself) is -// transient, not data loss — hydration's bounded missing-pack retry -// absorbs it — and the next sweep's heal pass remains the crash backstop. +// transient, not data loss: a registered pack has no bytes until the restore +// lands (one registry read plus one server-side copy), and a hydration that +// finds the same pack missing on consecutive listings waits between them +// (githttp.hydrateMissingWait) for that restore. The next sweep's heal pass +// remains the backstop when the restore fails or the sweeper dies. func (s *Sweeper) Sweep(ctx context.Context) (SweepReport, error) { report, err := s.sweep(ctx) // ADR-48: one outcome per execution, on the choke point both queue diff --git a/internal/jobs/sweep_window_test.go b/internal/jobs/sweep_window_test.go new file mode 100644 index 000000000..981e684af --- /dev/null +++ b/internal/jobs/sweep_window_test.go @@ -0,0 +1,251 @@ +// Sweeper-vs-writer window tests (claim B): a writer re-registers an orphan +// pack key between the sweeper's classify and its delete. The sweeper and the +// writer use real stores (sqlite registry, file blob store, real git packs). +// Decorators over blob.BlobStore and PackRegistry park the sweeper at chosen +// calls, so every interleaving here is deterministic. +package jobs + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/coder/openstorage/internal/blob" + "github.com/coder/openstorage/internal/githttp" + "github.com/coder/openstorage/internal/ref" +) + +// windowPack is one pack the writer publishes: its key, local file and the +// commit its ref move points at. +type windowPack struct { + id, key, path, commit string + size int64 +} + +// mintOrphanPack builds a real pack for a new commit, uploads it, ages it +// past the sweep threshold and leaves it unregistered: an orphan the sweeper +// will classify for deletion. The local pack file stays for the writer. +func mintOrphanPack(t *testing.T, f *repackFixture) windowPack { + t.Helper() + ctx := context.Background() + f.seq++ + blobSHA := strings.TrimSpace(testGit(t, f.scratch, fmt.Sprintf("window %d\n", f.seq), "hash-object", "-w", "--stdin")) + treeSHA := strings.TrimSpace(testGit(t, f.scratch, fmt.Sprintf("100644 blob %s\tdata.txt\n", blobSHA), "mktree")) + commitSHA := strings.TrimSpace(testGit(t, f.scratch, "", "commit-tree", treeSHA, "-m", fmt.Sprintf("w%d", f.seq))) + packDir := t.TempDir() + objs := blobSHA + "\n" + treeSHA + "\n" + commitSHA + "\n" + id := strings.TrimSpace(testGit(t, f.scratch, objs, "pack-objects", "--threads=1", "-q", filepath.Join(packDir, "w"))) + p := windowPack{id: id, key: blob.PackKey(f.repo, id), path: filepath.Join(packDir, "w-"+id+".pack"), commit: commitSHA} + fi, err := os.Stat(p.path) + if err != nil { + t.Fatalf("stat pack: %v", err) + } + p.size = fi.Size() + if err := f.blobs.UploadFile(ctx, p.key, p.path); err != nil { + t.Fatalf("upload orphan: %v", err) + } + past := time.Now().Add(-25 * time.Hour) + if err := os.Chtimes(filepath.Join(f.blobDir, filepath.FromSlash(p.key)), past, past); err != nil { + t.Fatalf("backdate orphan: %v", err) + } + return p +} + +// The writer below runs the sweeper-relevant calls of api.publishPack with +// quota off, one step per helper so a test can interleave them: the Stat at +// publish.go:58, RegisterPacks at :175, blob.EnsurePublished at :200, then +// the caller's ref move. + +// writerStat is publish.go:58. A present key is "published": no upload. +func writerStat(t *testing.T, f *repackFixture, p windowPack) (published bool) { + t.Helper() + _, err := f.blobs.Stat(context.Background(), p.key) + if err != nil && !errors.Is(err, blob.ErrNotFound) { + t.Fatalf("writer stat: %v", err) + } + return err == nil +} + +// writerRegister is publish.go:175 for a published (adopted) pack. +func writerRegister(t *testing.T, f *repackFixture, p windowPack) { + t.Helper() + pack := ref.Pack{ID: p.id, Key: p.key, Size: p.size, Adopted: true} + if err := f.store.RegisterPacks(context.Background(), f.repo, []ref.Pack{pack}); err != nil { + t.Fatalf("writer register: %v", err) + } +} + +// writerVerify is publish.go:200: re-upload when the key is missing. +func writerVerify(t *testing.T, f *repackFixture, p windowPack) (restored bool) { + t.Helper() + restored, err := blob.EnsurePublished(context.Background(), f.blobs, p.key, p.path, nil, nil) + if err != nil { + t.Fatalf("writer verify: %v", err) + } + return restored +} + +// writerMoveRef is the ref move the caller makes once publishPack returns. +func writerMoveRef(t *testing.T, f *repackFixture, p windowPack) { + t.Helper() + won, err := f.store.CASRef(context.Background(), f.repo, "refs/heads/main", f.tip, p.commit) + if err != nil || !won { + t.Fatalf("writer ref move: won=%v err=%v", won, err) + } + f.tip = p.commit +} + +// windowBlobs is the sweeper's blob store. It calls stop before the +// sweeper deletes key ("delete") and before it copies a backup back onto key +// ("restore"), and records when each inner call returned. +type windowBlobs struct { + blob.BlobStore + key string + stop func(step string) error + deletedAt, restoredAt time.Time +} + +func (b *windowBlobs) Delete(ctx context.Context, key string) error { + if key != b.key { + return b.BlobStore.Delete(ctx, key) + } + if err := b.stop("delete"); err != nil { + return err + } + err := b.BlobStore.Delete(ctx, key) + b.deletedAt = time.Now() + return err +} + +func (b *windowBlobs) Copy(ctx context.Context, dst, src string) error { + if dst != b.key { // the quarantine copy writes the backup key + return b.BlobStore.Copy(ctx, dst, src) + } + if err := b.stop("restore"); err != nil { + return err + } + err := b.BlobStore.Copy(ctx, dst, src) + b.restoredAt = time.Now() + return err +} + +// windowPacks is the sweeper's registry. Its third HasPack of key is +// safeDelete's post-delete recheck (1 = classify, 2 = revalidateOrphan), and +// it calls stop before that read. +type windowPacks struct { + PackRegistry + key string + calls int + stop func(step string) error +} + +func (p *windowPacks) HasPack(ctx context.Context, repo, key string) (bool, error) { + if key == p.key { + p.calls++ + if p.calls == 3 { + if err := p.stop("recheck"); err != nil { + return false, err + } + } + } + return p.PackRegistry.HasPack(ctx, repo, key) +} + +// parker parks the sweeper goroutine at a step until the test resumes it. +type parker struct { + at chan string + resume chan error +} + +func newParker() *parker { return &parker{at: make(chan string), resume: make(chan error)} } + +func (p *parker) stop(step string) error { + p.at <- step + return <-p.resume +} + +func (p *parker) expect(t *testing.T, step string) { + t.Helper() + select { + case got := <-p.at: + if got != step { + t.Fatalf("sweeper parked at %q, want %q", got, step) + } + case <-time.After(30 * time.Second): + t.Fatalf("sweeper never reached %q", step) + } +} + +func windowSweeper(f *repackFixture, key string, stop func(string) error) (*Sweeper, *windowBlobs) { + bl := &windowBlobs{BlobStore: f.blobs, key: key, stop: stop} + pk := &windowPacks{PackRegistry: f.store, key: key, stop: stop} + return NewSweeper(bl, pk, SweepConfig{Threshold: 24 * time.Hour, Delete: true}, slog.New(slog.DiscardHandler)), bl +} + +// Sweeper.Sweep's doc comment says that a reader racing the restore is +// transient. The hydration retry must wait for the restore: a cold hydrate that +// starts inside the window, with the restore 200 ms away, used to fail with +// ErrPackMissing within milliseconds, because the retry listed again 8 times +// with no wait. +func TestSweepWindow_ColdHydrateInWindowWaitsForRestore(t *testing.T) { + f := newRepackFixture(t) + f.push(t) + p := mintOrphanPack(t, f) + park := newParker() + s, _ := windowSweeper(f, p.key, park.stop) + done := make(chan struct{}) + go func() { defer close(done); _, _ = s.Sweep(context.Background()) }() + + park.expect(t, "delete") + writerStat(t, f, p) + writerRegister(t, f, p) + writerVerify(t, f, p) + writerMoveRef(t, f, p) + park.resume <- nil + park.expect(t, "recheck") + park.resume <- nil + park.expect(t, "restore") + + repoPath, err := githttp.EnsureCacheRepo(t.TempDir(), f.repo) + if err != nil { + t.Fatalf("EnsureCacheRepo: %v", err) + } + // The restore waits until the hydration has read the missing pack, so that + // the test always enters the window, whatever the scheduler does. + seen := ¬FoundSignal{BlobStore: f.blobs, seen: make(chan struct{})} + hydrated := make(chan error, 1) + go func() { hydrated <- githttp.HydratePacks(context.Background(), f.store, seen, repoPath, f.repo) }() + select { + case <-seen.seen: + case <-time.After(30 * time.Second): + t.Fatal("the hydration never read the missing pack") + } + time.Sleep(200 * time.Millisecond) // the restore is 200 ms away from the first miss + park.resume <- nil // the restore copy runs + <-done + if err := <-hydrated; err != nil { + t.Fatalf("cold hydrate that started 200ms before the sweeper's restore: %v", err) + } +} + +// notFoundSignal closes seen at the first download that finds no blob. +type notFoundSignal struct { + blob.BlobStore + once sync.Once + seen chan struct{} +} + +func (n *notFoundSignal) DownloadFile(ctx context.Context, key, path string) error { + err := n.BlobStore.DownloadFile(ctx, key, path) + if errors.Is(err, blob.ErrNotFound) { + n.once.Do(func() { close(n.seen) }) + } + return err +}