Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .agents/skills/verify-lane-schedule/model/fingerprints.txt
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,7 @@ internal/githttp/hydrate.go:holdFileIdentity 4451039fff307c74d3a416e209140826edc
internal/githttp/hydrate.go:holdLineCovers 68e0cad1251be7349a4179f64a25ab38c92ac2e473d6e986c078acaf556520fe
internal/githttp/hydrate.go:holdSidecarOwner d344d481c0a6961091973246ecd86c36a1adf26024a7f84daade1a3beb8b72ec
internal/githttp/hydrate.go:hydrateOnePack 65e2d6f356099992e8b9e49e8f22855fad64e053f8d4936e87b6ec14b4f382fa
internal/githttp/hydrate.go:hydratePacksBy c8410824050b8b8c2048395646cb00e2a04625bbfead2a0d30b6e0f819e837ca
internal/githttp/hydrate.go:hydratePacksBy 596858a2e0363c40e36ec6db3ef581324457041c17c5f9c42c863d33d88cc624
internal/githttp/hydrate.go:hydrateRefs 10c0aee5ef4b4352de3f3e1d03e899a86951f88b76538a038b6be16f3af840d3
internal/githttp/hydrate.go:import_"bufio" c8716a546ce61a4869563c666058adbec5f4783581ea4f74788f87ec1f0731f1
internal/githttp/hydrate.go:import_"bytes" 151fc787b0679fc8767e367d1971b9f907660a3d72d8fa3888877ef9f8ebd9c3
Expand Down Expand Up @@ -421,6 +421,8 @@ internal/githttp/hydrate.go:var_errConfigValueMismatch 6072c7d12edf5f49796836c2e
internal/githttp/hydrate.go:var_errPushScanBudget 21c09212ded082bfd4aaab0be40dfe2db6fb09bb6483778ea83d5d7112ff94be
internal/githttp/hydrate.go:var_gitSpawnObserver dad15a1d668f4997e06a2c42ecf9814ef4d288463c954ed8503701b311c4d761
internal/githttp/hydrate.go:var_hookVerifyBarrier dec9fba51ee077b370309f5ed4434c9a35961db2b11d69d72854955ca34c4be7
internal/githttp/hydrate.go:var_hydrateMissingWait 5e73ece8bff435c5d7eaa025de4a7845dd4c89afe3f7f4e23174fcb9569bae36
internal/githttp/hydrate.go:var_hydrateMissingWaitMax f0661515a42c0fbfa62d8057e46687d5b20c3606976a313e31a9734781e3cd5e
internal/githttp/hydrate.go:var_linkFile 47e0a13d524db684a6284b85a674578d5522836e5d22acb6b89909a1d44653d9
internal/githttp/hydrate.go:var_maxPushPeerScanBytes 66203545f4422b2f2c96c6b2ef37e9f1851a855c391a8405cff2b2288909701b
internal/githttp/hydrate.go:var_pushHoldTTL d3b2bcf904b9d88d9983388e6f79346bf8b8f4c6466047f7db4f4607ac6b91cb
Expand Down
27 changes: 26 additions & 1 deletion internal/githttp/hydrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -886,6 +886,18 @@ func ensureHook(path, script string) (os.FileInfo, error) {
// genuinely gone, not racing.
const maxHydrateRaces = 8

// hydrateMissingWait is the first wait between two listings that both find the
// same pack missing (it doubles each time, to hydrateMissingWaitMax). A fold
// that replaced the pack lists the replacement, so a pack that is listed and
// missing twice in a row is no fold race. The orphan sweep can delete the bytes
// of a pack that a writer registered just before, and restores them within one
// pass: a reader that starts inside that window waits for the restore instead
// of failing within milliseconds. Tests shorten it.
var (
hydrateMissingWait = 50 * time.Millisecond
hydrateMissingWaitMax = time.Second
)

// ErrPackMissing is returned (wrapped, match with errors.Is) when a
// registered pack's blob stays missing after maxHydrateRaces listing passes:
// the ref store references bytes that do not exist, which is corruption
Expand Down Expand Up @@ -925,12 +937,15 @@ func hydratePacksBy(ctx context.Context, bl blob.BlobStore, repoPath, repo strin
return err
}
var lastMissing error
for range maxHydrateRaces {
var prevMissingID string
wait := hydrateMissingWait
for attempt := range maxHydrateRaces {
packs, err := list(ctx)
if err != nil {
return err
}
lastMissing = nil
missingID := ""
for _, p := range packs {
base := filepath.Join(packDir, "pack-"+p.ID)
if _, err := os.Stat(base + ".idx"); err == nil {
Expand All @@ -941,12 +956,22 @@ func hydratePacksBy(ctx context.Context, bl blob.BlobStore, repoPath, repo strin
return err
}
lastMissing = fmt.Errorf("pack %s: %w", p.ID, err)
missingID = p.ID
break
}
}
if lastMissing == nil {
return nil
}
if missingID == prevMissingID && attempt < maxHydrateRaces-1 {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(wait):
}
wait = min(wait*2, hydrateMissingWaitMax)
Comment thread
ThomasK33 marked this conversation as resolved.
Resolved
}
prevMissingID = missingID
}
return fmt.Errorf("hydrate %s: %w after %d attempts: %v", repo, ErrPackMissing, maxHydrateRaces, lastMissing)
}
Expand Down
99 changes: 99 additions & 0 deletions internal/githttp/hydrate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -679,6 +679,7 @@ func TestHydratePacks_MissingPackExhaustsBound(t *testing.T) {
defer bl.Close()
gone := ref.Pack{ID: "gone", Key: blob.PackKey(repo, "gone")}
st := &scriptedPacks{lists: [][]ref.Pack{{gone}}}
shortenHydrateWait(t)

repoPath, err := EnsureCacheRepo(t.TempDir(), repo)
if err != nil {
Expand All @@ -696,6 +697,104 @@ func TestHydratePacks_MissingPackExhaustsBound(t *testing.T) {
}
}

// shortenHydrateWait makes the wait between two listings that find the same
// pack missing 1 ms for the test. The test must not run in parallel.
func shortenHydrateWait(t *testing.T) {
t.Helper()
oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax
hydrateMissingWait, hydrateMissingWaitMax = time.Millisecond, time.Millisecond
t.Cleanup(func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax })
}

// timedRestoreStore lists one pack, and starts a timer at the first listing that
// brings the pack's blob back: a sweeper that restores the pack's bytes.
type timedRestoreStore struct {
ref.RefStore
pack ref.Pack
after time.Duration
back func()
calls int
}

func (s *timedRestoreStore) ListPacks(context.Context, string) ([]ref.Pack, error) {
if s.calls++; s.calls == 1 {
time.AfterFunc(s.after, s.back)
}
return []ref.Pack{s.pack}, nil
}

// A pack that is listed and missing on consecutive listings is no fold race:
// the sweeper can have deleted the bytes of a pack that a writer registered, and
// restores them within one pass. The hydration waits between the listings for
// the restore. A pack that a fold replaced needs no wait.
func TestHydratePacks_WaitsForAPackThatStaysMissing(t *testing.T) {
ctx := context.Background()
const repo = "t-hydrate-wait"
bl, err := blob.OpenFile("file://" + t.TempDir())
if err != nil {
t.Fatalf("OpenFile: %v", err)
}
defer bl.Close()
back := ref.Pack{ID: "back", Key: blob.PackKey(repo, "back")}
oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax
hydrateMissingWait, hydrateMissingWaitMax = 100*time.Millisecond, 100*time.Millisecond
defer func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }()
packFile := buildTestPack(t)
repoPath, err := EnsureCacheRepo(t.TempDir(), repo)
if err != nil {
t.Fatalf("EnsureCacheRepo: %v", err)
}
// The blob comes back 250 ms after the first listing: more than the 8
// listings take without a wait, less than the waits cover.
st := &timedRestoreStore{pack: back, after: 250 * time.Millisecond, back: func() { _ = bl.UploadFile(ctx, back.Key, packFile) }}
if err := HydratePacks(ctx, st, bl, repoPath, repo); err != nil {
t.Fatalf("hydrate of a pack that comes back within the wait: %v", err)
}
if st.calls < 3 {
t.Fatalf("ListPacks called %d times, want at least 3 (the wait covers the first two misses)", st.calls)
}

// The wait ends with the context.
gone := ref.Pack{ID: "gone", Key: blob.PackKey(repo, "gone")}
hydrateMissingWait, hydrateMissingWaitMax = time.Hour, time.Hour
cctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
defer cancel()
start := time.Now()
err = HydratePacks(cctx, &scriptedPacks{lists: [][]ref.Pack{{gone}}}, bl, repoPath, repo)
if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 5*time.Second {
t.Fatalf("hydrate with a context that ends = %v after %v, want the context error soon", err, time.Since(start))
}
}

// A fold race retries at once: the second listing carries the replacement, so
// the missing pack differs and the hydration does not wait.
func TestHydratePacks_FoldRaceRetriesWithoutWaiting(t *testing.T) {
const repo = "t-hydrate-fold-fast"
bl, err := blob.OpenFile("file://" + t.TempDir())
if err != nil {
t.Fatalf("OpenFile: %v", err)
}
defer bl.Close()
replacement := ref.Pack{ID: "new", Key: blob.PackKey(repo, "new")}
if err := bl.UploadFile(context.Background(), replacement.Key, buildTestPack(t)); err != nil {
t.Fatalf("UploadFile: %v", err)
}
folded := ref.Pack{ID: "old", Key: blob.PackKey(repo, "old")}
oldWait, oldMax := hydrateMissingWait, hydrateMissingWaitMax
hydrateMissingWait, hydrateMissingWaitMax = time.Hour, time.Hour
defer func() { hydrateMissingWait, hydrateMissingWaitMax = oldWait, oldMax }()
repoPath, err := EnsureCacheRepo(t.TempDir(), repo)
if err != nil {
t.Fatalf("EnsureCacheRepo: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
st := &scriptedPacks{lists: [][]ref.Pack{{folded}, {replacement}}}
if err := HydratePacks(ctx, st, bl, repoPath, repo); err != nil || st.calls != 2 {
t.Fatalf("hydrate across a fold = %v after %d listings, want success after 2 with no wait", err, st.calls)
}
}

// Transport errors must keep failing fast: no re-list, no retry, and no
// misclassification as a missing pack.
func TestHydratePacks_TransportErrorFailsFast(t *testing.T) {
Expand Down
7 changes: 5 additions & 2 deletions internal/jobs/sweep.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,8 +322,11 @@ func NewSweeper(blobs blob.BlobStore, packs PackRegistry, cfg SweepConfig, logge
// assumptions (see EnsurePublished's ordering proof).
//
// Any residual exposure (e.g. a reader racing the restore itself) is
// transient, not data loss — hydration's bounded missing-pack retry
// absorbs it — and the next sweep's heal pass remains the crash backstop.
// transient, not data loss: a registered pack has no bytes until the restore
// lands (one registry read plus one server-side copy), and a hydration that
// finds the same pack missing on consecutive listings waits between them
// (githttp.hydrateMissingWait) for that restore. The next sweep's heal pass
// remains the backstop when the restore fails or the sweeper dies.
func (s *Sweeper) Sweep(ctx context.Context) (SweepReport, error) {
report, err := s.sweep(ctx)
// ADR-48: one outcome per execution, on the choke point both queue
Expand Down
Loading
Loading