diff --git a/.agents/skills/verify-lane-schedule/SKILL.md b/.agents/skills/verify-lane-schedule/SKILL.md index 6551defc..2833d3e9 100644 --- a/.agents/skills/verify-lane-schedule/SKILL.md +++ b/.agents/skills/verify-lane-schedule/SKILL.md @@ -226,7 +226,7 @@ With every fact TRUE in the code, setting each fact FALSE in the model alone fai ## Coverage record -Sync commit: `0f5979dd0589da6224230f74e9bc7d15143ef5d1` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). +Sync commit: `e1d4a797802da859205fc67494aa719ff23e68a3` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). Every declaration in these files is covered: diff --git a/.agents/skills/verify-lane-schedule/model/fingerprints.txt b/.agents/skills/verify-lane-schedule/model/fingerprints.txt index f9cdaaa6..c6175d80 100644 --- a/.agents/skills/verify-lane-schedule/model/fingerprints.txt +++ b/.agents/skills/verify-lane-schedule/model/fingerprints.txt @@ -181,6 +181,7 @@ internal/githttp/hook.go:const_reservationDenied f72c8a7d40a9207271da47cf160c52e internal/githttp/hook.go:const_reservationJIT d051ee28340ba848f8282f31689c8bbdde129b4970df2be75292d3b4486c429c internal/githttp/hook.go:const_reservationNeedsVerify b8ec7eeca735e84bb9aa7aa596b7961d17778741956392c03d71116285ff5730 internal/githttp/hook.go:denyAuthorityDFConflict 3ad4a9b7cd317dbae7ebf368256b119a33774f58734c26fef36d47545f50ba5b +internal/githttp/hook.go:denyUnregisteredTarget a07930a318a3bd3488190e6869d6ba1d7589354257c8b832d66e43b733ad665c internal/githttp/hook.go:enforceRefPolicy 2cd2cf6aea243221e8fe27280a6333695d4455a0304e7f536f7e8729dbb17a4d internal/githttp/hook.go:enforceSyncClaimFence efb7b07e54bbb4008ac0cf53778c2ba690ad936051c103421aaf308b52e1cdf9 internal/githttp/hook.go:evalRefPolicy b232fd0a747ced157cec1bc4588e53bc13c54a39ff829f45108ae62c9601b5c8 @@ -248,7 +249,7 @@ internal/githttp/hook.go:type_refValuesProber 0389595e3fb341c7e67784d27674d922dd internal/githttp/hook.go:type_reservationClass 48d346b5a457165c93ed2a9aca43bcd39438f05201427fa2351d8b3a4066ff29 internal/githttp/hook.go:type_sigVerifier afbcc825325e58413e4a3646073bab61862ebda41b2c8c9f6fefae2ad35a4745 internal/githttp/hook.go:updateBarrierEachPath 53afd6c803fba2472970b4a81e706f4e6c975c84753c0c94714bd52c1fdd0fcf -internal/githttp/hook.go:updateHook 2d7d924d17781fd711f5c8395f2b0e3d4511ef5e12e4643ef8c8716a607f670f +internal/githttp/hook.go:updateHook a502272c8ecb19db2bf09a988ed121d8b7be88f4fff859137bf1714f60e0cb6d internal/githttp/hook.go:var_WrapHookRefStore 2a84a08163beaeaef666d54ff198e62e68b8e70a83fa6f46f9025b30c28ce5c5 internal/githttp/hook.go:var__ 14f3ac5e5fb70c7fe12bcffeaa3f5891a5f61b624d170427d94afa8153dcf5fd internal/githttp/hook.go:var__#2 d54a83bcc2c7fb6caf4b5040f8c28ca780368fb5c2c578f05c5a48f6edb6dd97 @@ -268,6 +269,7 @@ internal/githttp/hook.go:var_pushRepoDirLen 9db02c4285217113dfb520f541a4af97e86a internal/githttp/hook.go:var_refPolicyCache 12635e37a2a1162463fb6f64c3fa934b9e3793394ff7847bc17bf2015a822e50 internal/githttp/hook.go:var_refPolicyParses e1a83d07f4b25c69151a945c0c6abb1110dd5710769f8c3c8296824f19c1d01b internal/githttp/hook.go:var_refTxFlushAt f6baf6b2e0ccd94a65e622a30218ecf8e574c22c94141b0b99cc4fb2e01e55aa +internal/githttp/hook.go:var_unregisteredTargetCheck c3c0d603df53454230d740514760d74554900d1fb3fbc2d052036fed07e03802 internal/githttp/hook.go:verifySig 51a7f42302b43ac7155d125dd99bbaf3abd305fb30512c5357230bb333414778 internal/githttp/hook.go:verifySigDenied 478714e01ee647624c46ac6f5b5aa04c2c7bac29c61f3cee57bdf7bdc0142a56 internal/githttp/hydrate.go:BaseRepo 20aab83202d8a04a36232730cb5eb3f34f38261ec029f26db7bc44677aeb6a20 diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index 10a7e092..0dae4739 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -687,6 +687,15 @@ old behavior, read `default_branch` from `GET /repo` and send it as `base_ref`. This follows the public OpenAPI/SDK contract, not demonstrated upstream cluster behavior ([contract audit](../explanation/wire-compatibility.md#server-allocated-branch-contract)). +#### Which commit the base and the tag target can name + +The commit that `base_ref`, `base_branch` or `preferred_base` of `POST /repos/branches/create`, or +`ref` of `POST /repos/tags`, names must be in a pack that the repository's pack registry lists. +The server checks the registered packs, not the node's cache repository, because every other node +(and this node after it loses its cache) hydrates from the registered packs alone and cannot hold a +ref at any other object. A commit that only a cache holds answers `404`, like a commit that does +not exist, and a `base_ref` that names no usable commit still falls back to `preferred_base`. + #### Guarded branch create and move `expected_target_sha` on `POST /repos/branches/create` (both path styles) guards the diff --git a/internal/api/handlers_write_refs.go b/internal/api/handlers_write_refs.go index f04e92f5..5d96bf63 100644 --- a/internal/api/handlers_write_refs.go +++ b/internal/api/handlers_write_refs.go @@ -386,6 +386,29 @@ func wrefValidateCreateBranch(req *wrefCreateBranchRequest) (target string, allo return target, allocate, guard, errs.err() } +// wrefResolveHeld is gitread.ResolveRef for a write that stores the commit in a +// ref. The commit must also be in a pack that the registry lists for the +// repository: the node's cache can hold objects that no registered pack holds +// (a cache that was copied or edited out of band, or a registry that was rolled +// back), and a ref at such an object wedges every other node and this node +// after its cache is gone, because they hydrate from the registered packs +// alone. A commit that no registered pack holds answers like one that does not +// exist. +func (s *server) wrefResolveHeld(ctx context.Context, repoID, repoPath, spec string, ephemeral bool) (string, error) { + sha, err := gitread.ResolveRef(ctx, repoPath, spec, ephemeral) + if err != nil { + return "", err + } + kinds, err := gitread.RegisteredKinds(ctx, s.refStore, s.blobStore, repoPath, repoID, []string{sha}) + if err != nil { + return "", fmt.Errorf("check that a registered pack holds %s: %w", sha, err) + } + if kinds[sha] != "commit" { + return "", gitread.ErrNotFound + } + return sha, nil +} + // handleCreateBranch resolves the base to a commit (branch, tag — peeled — // or hex sha via gitread.ResolveRef; preferred_base is the fallback when // the primary base is absent or names no ref, and a base whose commit git @@ -436,7 +459,7 @@ func (s *server) handleCreateBranch(w http.ResponseWriter, r *http.Request) { ctx := r.Context() baseSHA := "" if base != "" { - sha, err := gitread.ResolveRef(ctx, repoPath, base, req.BaseIsEphemeral) + sha, err := s.wrefResolveHeld(ctx, repo.RepoID, repoPath, base, req.BaseIsEphemeral) switch { case errors.Is(err, gitread.ErrNotFound): // Fall through to preferred_base below (when given). An @@ -449,7 +472,7 @@ func (s *server) handleCreateBranch(w http.ResponseWriter, r *http.Request) { } } if baseSHA == "" && req.PreferredBase != "" { - sha, err := gitread.ResolveRef(ctx, repoPath, req.PreferredBase, req.BaseIsEphemeral) + sha, err := s.wrefResolveHeld(ctx, repo.RepoID, repoPath, req.PreferredBase, req.BaseIsEphemeral) switch { case errors.Is(err, gitread.ErrNotFound): // Report below with the primary-base message. @@ -752,7 +775,7 @@ func (s *server) handleCreateTag(w http.ResponseWriter, r *http.Request) { if !ok { return } - sha, err := gitread.ResolveRef(r.Context(), repoPath, target, false) + sha, err := s.wrefResolveHeld(r.Context(), repo.RepoID, repoPath, target, false) switch { case errors.Is(err, gitread.ErrNotFound): writeProblem(w, http.StatusNotFound, codeRefNotFound, "ref does not resolve to a commit") diff --git a/internal/api/registry_kinds_test.go b/internal/api/registry_kinds_test.go new file mode 100644 index 00000000..95a9cd62 --- /dev/null +++ b/internal/api/registry_kinds_test.go @@ -0,0 +1,72 @@ +package api + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/coder/openstorage/internal/githttp" + "github.com/coder/openstorage/internal/gitread" +) + +// RegisteredKinds answers from the registered packs alone: a commit that only +// the cache holds reads as missing, and a registered pack that the cache lacks +// (a fold replaced the packs since the cache was hydrated) is downloaded first, +// so that a healthy object never reads as missing. +func TestRegisteredKindsReadsTheRegisteredPacksOnly(t *testing.T) { + runBackends(t, func(t *testing.T, e *testEnv) { + fx := wrefSeedFixture(t, e) + ctx := context.Background() + repoPath, err := gitread.Materialize(ctx, e.refStore, e.blobStore, e.cacheDir, fx.repoID, "main") + if err != nil { + t.Fatal(err) + } + planted := plantCacheOnlyCommit(t, repoPath, fx.c2) + kinds, err := gitread.RegisteredKinds(ctx, e.refStore, e.blobStore, repoPath, fx.repoID, []string{fx.c1, fx.c2, fx.tagObj, planted}) + if err != nil { + t.Fatal(err) + } + want := map[string]string{fx.c1: "commit", fx.c2: "commit", fx.tagObj: "tag", planted: "missing"} + for id, k := range want { + if kinds[id] != k { + t.Fatalf("kind of %s = %q, want %q (all: %v)", id, kinds[id], k, kinds) + } + } + + // The cache loses its pack files: the registry still lists the packs. + packs, err := filepath.Glob(filepath.Join(repoPath, "objects", "pack", "pack-*")) + if err != nil || len(packs) == 0 { + t.Fatalf("pack files in the cache: %v, %v", packs, err) + } + for _, p := range packs { + if err := os.Remove(p); err != nil { + t.Fatal(err) + } + } + kinds, err = gitread.RegisteredKinds(ctx, e.refStore, e.blobStore, repoPath, fx.repoID, []string{fx.c2}) + if err != nil || kinds[fx.c2] != "commit" { + t.Fatalf("after the cache lost its packs: %v, %v", kinds, err) + } + }) +} + +// With no way to fetch a pack that the cache lacks, the view refuses, so that +// a caller never reads "missing" for an object that a registered pack holds. +func TestRegisteredPackViewRefusesWhenAPackIsMissingAndCannotBeFetched(t *testing.T) { + e := testEnvironment(t, "sqlite") + fx := wrefSeedFixture(t, e) + repoPath := filepath.Join(t.TempDir(), "empty.git") + if err := os.MkdirAll(filepath.Join(repoPath, "objects", "pack"), 0o755); err != nil { + t.Fatal(err) + } + if _, err := githttp.NewRegisteredPackView(context.Background(), e.refStore, repoPath, fx.repoID, nil); !errors.Is(err, githttp.ErrPackMissing) { + t.Fatalf("NewRegisteredPackView with no heal = %v, want ErrPackMissing", err) + } + calls := 0 + _, err := githttp.NewRegisteredPackView(context.Background(), e.refStore, repoPath, fx.repoID, func(context.Context) error { calls++; return nil }) + if !errors.Is(err, githttp.ErrPackMissing) || calls != 1 { + t.Fatalf("NewRegisteredPackView with a heal that fetches nothing = %v after %d heals, want ErrPackMissing after 1", err, calls) + } +} diff --git a/internal/api/registry_target_test.go b/internal/api/registry_target_test.go new file mode 100644 index 00000000..7ff7e7d8 --- /dev/null +++ b/internal/api/registry_target_test.go @@ -0,0 +1,106 @@ +package api + +// A ref write must not store a commit that only the node's cache holds. The +// cache can hold objects that no registered pack holds (a cache that was copied +// or edited out of band, or a registry that was rolled back), and a ref at such +// an object wedges every other node, and this node after its cache is gone, +// because they hydrate from the registered packs alone. + +import ( + "context" + "net/http" + "os" + "os/exec" + "strings" + "testing" + + "github.com/coder/openstorage/internal/gitread" +) + +// plantCacheOnlyCommit writes a commit (a child of parent, with a new blob) as +// loose objects into the cache repository at repoPath only: no pack holds it. +func plantCacheOnlyCommit(t *testing.T, repoPath, parent string) string { + t.Helper() + run := func(stdin string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repoPath + for _, kv := range os.Environ() { + if !strings.HasPrefix(kv, "GIT_") { + cmd.Env = append(cmd.Env, kv) + } + } + cmd.Env = append(cmd.Env, "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null", + "GIT_AUTHOR_NAME=p", "GIT_AUTHOR_EMAIL=p@x.invalid", "GIT_COMMITTER_NAME=p", "GIT_COMMITTER_EMAIL=p@x.invalid") + cmd.Stdin = strings.NewReader(stdin) + out, err := cmd.Output() + if err != nil { + t.Fatalf("git %v: %v", args, err) + } + return strings.TrimSpace(string(out)) + } + blob := run("cache only\n", "hash-object", "-w", "--stdin") + tree := run("100644 blob "+blob+"\tcacheonly.txt\n", "mktree") + return run("", "commit-tree", "-p", parent, "-m", "cache only", tree) +} + +func TestRefWritesRefuseACommitThatOnlyTheCacheHolds(t *testing.T) { + type shape struct { + name, path string + body func(sha string) map[string]any + ref string // the ref that the request would create + } + shapes := []shape{ + {"branch base_ref", "/api/v1/repos/branches/create", func(sha string) map[string]any { + return map[string]any{"target_branch": "co", "base_ref": sha} + }, "refs/heads/co"}, + {"branch preferred_base", "/api/v1/repos/branches/create", func(sha string) map[string]any { + return map[string]any{"target_branch": "co", "base_ref": "nosuch", "preferred_base": sha} + }, "refs/heads/co"}, + {"branch force", "/api/v1/repos/branches/create", func(sha string) map[string]any { + return map[string]any{"target_branch": "feature", "base_ref": sha, "force": true} + }, "refs/heads/feature"}, + {"ephemeral branch", "/api/v1/repos/branches/create", func(sha string) map[string]any { + return map[string]any{"target_branch": "co", "base_ref": sha, "base_is_ephemeral": true, "target_is_ephemeral": true} + }, "refs/namespaces/ephemeral/refs/heads/co"}, + {"tag", "/api/v1/repos/tags", func(sha string) map[string]any { + return map[string]any{"name": "co", "ref": sha} + }, "refs/tags/co"}, + } + runBackends(t, func(t *testing.T, e *testEnv) { + for _, tc := range shapes { + t.Run(tc.name, func(t *testing.T) { + fx := wrefSeedFixture(t, e) + repoPath, err := gitread.Materialize(context.Background(), e.refStore, e.blobStore, e.cacheDir, fx.repoID, "main") + if err != nil { + t.Fatal(err) + } + planted := plantCacheOnlyCommit(t, repoPath, fx.c2) + srv := newWrefServer(t, e) + before := wrefRefs(t, e, fx.repoID) + + resp, body := wrefDo(t, srv, http.MethodPost, tc.path, fx.token, tc.body(planted)) + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("a commit that only the cache holds: status %d, want 404, body %v", resp.StatusCode, body) + } + after := wrefRefs(t, e, fx.repoID) + if len(after) != len(before) || after[tc.ref] != before[tc.ref] { + t.Fatalf("refs changed after the refused request: before %v, after %v", before, after) + } + // Every other node hydrates from the registered packs alone. + if _, err := gitread.Materialize(context.Background(), e.refStore, e.blobStore, t.TempDir(), fx.repoID, "main"); err != nil { + t.Fatalf("a fresh node cannot hydrate the repository: %v", err) + } + + // A commit that a registered pack holds still works. + resp, body = wrefDo(t, srv, http.MethodPost, tc.path, fx.token, tc.body(fx.c1)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("a commit that a pack holds: status %d, want 200, body %v", resp.StatusCode, body) + } + if got := wrefRefs(t, e, fx.repoID)[tc.ref]; got != fx.c1 { + t.Fatalf("%s = %q, want %s", tc.ref, got, fx.c1) + } + }) + } + }) +} diff --git a/internal/githttp/hook.go b/internal/githttp/hook.go index adc2ffba..e96a3f4b 100644 --- a/internal/githttp/hook.go +++ b/internal/githttp/hook.go @@ -1840,6 +1840,21 @@ func updateHook(ctx context.Context, args []string) (retErr error) { return err } } + // The new value must be in a pack that the registry lists. git has already + // checked that the object is readable, but it reads the node's cache, which + // can hold objects that no registered pack holds (a cache that was copied or + // edited out of band, or a registry that was rolled back). A ref at such an + // object wedges every node that hydrates from the registered packs. A push + // that carries its objects is safe: pre-receive registered its pack before + // this hook runs, and git moved the pack into the cache. The case that this + // closes is an update to an object that no pack of this push carries. + // Checked on the raw store, before the CAS, and denied per ref like a policy + // denial. Deletions have no new value. + if new != ref.ZeroSHA { + if err := unregisteredTargetCheck(ctx, st, repo, refName, new); err != nil { + return err + } + } var inner ref.RefStore = st if testMode() { inner = injectCASDrop(inner) @@ -2917,6 +2932,51 @@ func objectExists(sha string) bool { panic(fmt.Sprintf("githttp assertion failed: cat-file -e %s: %v", sha, err)) } +// unregisteredTargetCheck is denyUnregisteredTarget. Tests that run the update +// hook on an object id that they invented, to reach a later step, replace it. +var unregisteredTargetCheck = denyUnregisteredTarget + +// denyUnregisteredTarget refuses the update of refName to new when no pack that +// the registry lists for repo holds the object. It reads the cache repository +// through a RegisteredPackView, so only the registered packs answer. A registered +// pack that the cache lacks (a fold replaced the packs after the cache was +// hydrated) is downloaded first. An object that is no commit is left to the +// other checks: only "missing" is refused here. +func denyUnregisteredTarget(ctx context.Context, st ref.RefStore, repo, refName, new string) error { + repoPath, err := os.Getwd() // git runs hooks in the repository + if err != nil { + return fmt.Errorf("update: find the repository: %w", err) + } + view, err := NewRegisteredPackView(ctx, st, repoPath, repo, func(ctx context.Context) error { + bl, err := blob.Open(ctx, mustEnv(envHookBlobURL...)) + if err != nil { + return err + } + defer bl.Close() + return HydratePacks(ctx, st, bl, repoPath, repo) + }) + if err != nil { + return fmt.Errorf("update: read the registered packs: %w", err) + } + defer view.Close() + var out strings.Builder + cmd := exec.CommandContext(ctx, "git", "cat-file", "--batch-check=%(objecttype)") + cmd.Env = gitlimit.NoReplaceObjects(view.Env(os.Environ())) + cmd.Stdin = strings.NewReader(new + "\n") + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return fmt.Errorf("update: read the object type of %s in the registered packs: %w", new, err) + } + kinds, err := ParseObjectKinds([]string{new}, out.String()) + if err != nil { + return fmt.Errorf("update: %w", err) + } + if kinds[new] == "missing" { + return refusef("update: %s is at %s, which no pack of this repository's registry holds", refName, new) + } + return nil +} + // objectInMainStore reports whether sha is resolvable WITHOUT the // quarantine. git runs pre-receive with GIT_OBJECT_DIRECTORY pointed at the // quarantine and the main store as an alternate (githooks(5)); stripping diff --git a/internal/githttp/hookcanary_test.go b/internal/githttp/hookcanary_test.go index c8d92906..028e1e2f 100644 --- a/internal/githttp/hookcanary_test.go +++ b/internal/githttp/hookcanary_test.go @@ -65,6 +65,16 @@ func canarySQLExec(t *testing.T, dbURL string, stmts ...string) { } } +// skipRegisteredTargetCheck lets a test run the update hook on an object id +// that it invented: the check that a registered pack holds the new value would +// refuse it before the step that the test wants to reach. +func skipRegisteredTargetCheck(t *testing.T) { + t.Helper() + prev := unregisteredTargetCheck + unregisteredTargetCheck = func(context.Context, ref.RefStore, string, string, string) error { return nil } + t.Cleanup(func() { unregisteredTargetCheck = prev }) +} + // canaryTable is a unique name for a table that does not exist. func canaryTable() string { return "canary_" + strings.ToLower(rand.Text()[:8]) } @@ -118,6 +128,7 @@ func TestUpdateHookHidesStepErrors(t *testing.T) { t.Run("WrapHookRefStore error", func(t *testing.T) { canary := canaryTable() records := hookEnv(t, canarySQLite(t)) + skipRegisteredTargetCheck(t) prev := WrapHookRefStore t.Cleanup(func() { WrapHookRefStore = prev }) WrapHookRefStore = func(context.Context, ref.RefStore) (ref.RefStore, error) { diff --git a/internal/githttp/hookrefusal_test.go b/internal/githttp/hookrefusal_test.go index dcb77446..741046c0 100644 --- a/internal/githttp/hookrefusal_test.go +++ b/internal/githttp/hookrefusal_test.go @@ -154,6 +154,7 @@ func TestUpdateHookHidesStoreErrors(t *testing.T) { st.Close() c := pgCanary() records := hookEnv(t, dbURL) + skipRegisteredTargetCheck(t) prev := WrapHookRefStore t.Cleanup(func() { WrapHookRefStore = prev }) WrapHookRefStore = func(_ context.Context, st ref.RefStore) (ref.RefStore, error) { diff --git a/internal/githttp/registryview.go b/internal/githttp/registryview.go new file mode 100644 index 00000000..ee1ec707 --- /dev/null +++ b/internal/githttp/registryview.go @@ -0,0 +1,138 @@ +package githttp + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/coder/openstorage/internal/ref" +) + +// A node's cache repository can hold an object that no registered pack holds: +// a cache that was copied, restored or edited out of band, or a registry that +// was rolled back. git reads such an object without complaint, so "git can +// read it" proves nothing about the repository's durable state. Every other +// node, and this node after its cache is gone, hydrates from the registered +// packs alone and cannot write a ref at that object. +// +// RegisteredPackView is the repository as those nodes see it: an object +// directory that holds exactly the packs that the registry lists. A git child +// that runs with GIT_OBJECT_DIRECTORY set to it, and with no alternates, reads +// an object only when a registered pack holds it. + +// RegisteredPackView is a temporary object directory that links the pack files +// of the repository's cache that the registry lists, and nothing else. +type RegisteredPackView struct { + dir string + repoPath string +} + +// NewRegisteredPackView lists the packs that st has for repo and links their +// files from repoPath's object directory. A registered pack that is not in the +// cache (a fold replaced the packs since the cache was last hydrated) is +// fetched by heal, once, and the listing is read again. A nil heal, or a pack +// that is still missing after it, is an error that wraps ErrPackMissing: the +// view would lack objects that the registry holds, and a caller that read +// "missing" from it would refuse a ref that is healthy. +func NewRegisteredPackView(ctx context.Context, st ref.RefStore, repoPath, repo string, heal func(context.Context) error) (*RegisteredPackView, error) { + packDir := filepath.Join(repoPath, "objects", "pack") + var packs []ref.Pack + for attempt := 0; ; attempt++ { + var err error + if packs, err = st.ListPacks(ctx, repo); err != nil { + return nil, fmt.Errorf("list the packs of %s: %w", repo, err) + } + missing := "" + for _, p := range packs { + if !packFilesExist(packDir, p.ID) { + missing = p.ID + break + } + } + if missing == "" { + break + } + if heal == nil || attempt > 0 { + return nil, fmt.Errorf("%w: the cache of %s lacks the registered pack %s", ErrPackMissing, repo, missing) + } + if err := heal(ctx); err != nil { + return nil, fmt.Errorf("fetch the registered packs of %s: %w", repo, err) + } + } + dir, err := os.MkdirTemp("", "openstorage-registry-view-") + if err != nil { + return nil, err + } + v := &RegisteredPackView{dir: dir, repoPath: repoPath} + viewPacks := filepath.Join(dir, "pack") + if err := os.Mkdir(viewPacks, 0o700); err != nil { + v.Close() + return nil, err + } + for _, p := range packs { + for _, ext := range []string{".pack", ".idx"} { + name := "pack-" + p.ID + ext + if err := os.Symlink(filepath.Join(packDir, name), filepath.Join(viewPacks, name)); err != nil { + v.Close() + return nil, err + } + } + } + return v, nil +} + +func packFilesExist(packDir, id string) bool { + for _, ext := range []string{".pack", ".idx"} { + if _, err := os.Stat(filepath.Join(packDir, "pack-"+id+ext)); err != nil { + return false + } + } + return true +} + +// Close removes the view. The pack files that it linked stay. +func (v *RegisteredPackView) Close() { + if v != nil && v.dir != "" { + _ = os.RemoveAll(v.dir) + } +} + +// Env returns base with the variables that point git at the cache repository +// and at the view, and none of git's other repository variables. +func (v *RegisteredPackView) Env(base []string) []string { + out := make([]string, 0, len(base)+2) + for _, kv := range base { + name, _, _ := strings.Cut(kv, "=") + switch name { + case "GIT_DIR", "GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_QUARANTINE_PATH": + default: + out = append(out, kv) + } + } + return append(out, "GIT_DIR="+v.repoPath, "GIT_OBJECT_DIRECTORY="+v.dir) +} + +// ParseObjectKinds reads the output of `git cat-file +// --batch-check=%(objecttype)` for ids: one line for each id, the object's +// type or " missing". +func ParseObjectKinds(ids []string, out string) (map[string]string, error) { + lines := strings.Split(strings.TrimSuffix(out, "\n"), "\n") + if len(lines) != len(ids) { + return nil, fmt.Errorf("cat-file --batch-check answered %d lines for %d ids", len(lines), len(ids)) + } + kinds := make(map[string]string, len(ids)) + for i, l := range lines { + switch l { + case "commit", "tree", "blob", "tag": + kinds[ids[i]] = l + case ids[i] + " missing": + kinds[ids[i]] = "missing" + default: + return nil, errors.New("cat-file --batch-check answered an unknown line for " + ids[i]) + } + } + return kinds, nil +} diff --git a/internal/githttp/registryview_test.go b/internal/githttp/registryview_test.go new file mode 100644 index 00000000..293c8b8a --- /dev/null +++ b/internal/githttp/registryview_test.go @@ -0,0 +1,31 @@ +package githttp + +import ( + "slices" + "strings" + "testing" +) + +func TestParseObjectKinds(t *testing.T) { + a, b, c := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40) + got, err := ParseObjectKinds([]string{a, b, c}, "commit\n"+b+" missing\ntag\n") + if err != nil || got[a] != "commit" || got[b] != "missing" || got[c] != "tag" { + t.Fatalf("ParseObjectKinds = %v, %v", got, err) + } + for _, bad := range []string{"commit\n", "commit\nbogus\ntag\n", "commit\n" + c + " missing\ntag\n"} { + if _, err := ParseObjectKinds([]string{a, b, c}, bad); err == nil { + t.Fatalf("ParseObjectKinds(%q) = nil error, want a refusal", bad) + } + } +} + +// The view points git at the cache repository and at the view directory, and at +// no alternate or quarantine that the caller's environment names. +func TestRegisteredPackViewEnvReplacesTheObjectVariables(t *testing.T) { + v := &RegisteredPackView{dir: "/view", repoPath: "/cache/r.git"} + env := v.Env([]string{"PATH=/bin", "GIT_DIR=.", "GIT_OBJECT_DIRECTORY=/q", "GIT_ALTERNATE_OBJECT_DIRECTORIES=/main", "GIT_QUARANTINE_PATH=/q", "GIT_COMMON_DIR=/c", "HOME=/h"}) + want := []string{"PATH=/bin", "HOME=/h", "GIT_DIR=/cache/r.git", "GIT_OBJECT_DIRECTORY=/view"} + if !slices.Equal(env, want) { + t.Fatalf("Env = %v, want %v", env, want) + } +} diff --git a/internal/githttp/s3wal_opcensus_test.go b/internal/githttp/s3wal_opcensus_test.go index 6eb8f274..1d714e18 100644 --- a/internal/githttp/s3wal_opcensus_test.go +++ b/internal/githttp/s3wal_opcensus_test.go @@ -209,7 +209,10 @@ func TestS3WAL_PushOpCensus(t *testing.T) { // update hook's format check before its CAS (issue #445): one GET of // _wal/meta.json, so a wrong metadata format refuses the update instead // of dropping its events. The post-CAS reads then revalidate with 304s. - const steadyBudget = 21 + // 22 = the update hook's check that a registered pack holds the new value + // (denyUnregisteredTarget): one listing of the registered packs, a manifest + // read. A delete has no new value, so the delete-only budget is unchanged. + const steadyBudget = 22 if len(steady) > steadyBudget { t.Errorf("steady-state push cost %d S3 round trips, budget is %d:\n%s", len(steady), steadyBudget, censusTable(steady)) diff --git a/internal/githttp/unregistered_target_e2e_test.go b/internal/githttp/unregistered_target_e2e_test.go new file mode 100644 index 00000000..25c79122 --- /dev/null +++ b/internal/githttp/unregistered_target_e2e_test.go @@ -0,0 +1,64 @@ +package githttp + +// A node's cache repository can hold an object that no registered pack holds (a +// cache that was copied or edited out of band, or a registry that was rolled +// back). git reads such an object, so its connectivity check lets a push update +// a ref to it, with an empty pack that carries nothing. Every other node +// hydrates from the registered packs alone and cannot write a ref at that +// object, so the ref would wedge the repository. The update hook refuses it. + +import ( + "os/exec" + "strings" + "testing" + + "github.com/coder/openstorage/internal/ref" +) + +func TestUpdateHookRefusesATargetThatNoRegisteredPackHolds(t *testing.T) { + cache := t.TempDir() + node, err := startNode(cache) + if err != nil { + t.Fatalf("start node: %v", err) + } + t.Cleanup(node.stop) + repo := registerRepo(t, "t-unregistered-target") + work := initWorkRepo(t) + token := mintTestToken(t, repo, "git:read,git:write", "") + mustGit(t, work, "push", remoteURL(node, repo, token), "HEAD:refs/heads/main") + c1 := revParse(t, work, "HEAD") + tree := revParse(t, work, c1+"^{tree}") + + // The node's cache holds a commit that no registered pack holds. Its tree + // and parent are in a registered pack, so git's connectivity check passes. + cacheOnly := plantCommit(t, RepoPath(cache, repo), tree, c1) + cmd := exec.Command("git", "pack-objects", "--stdout", "-q") + cmd.Dir = work + emptyPack, err := cmd.Output() // no object ids on stdin: a pack with no objects + if err != nil { + t.Fatalf("git pack-objects of nothing: %v", err) + } + + const refused = "no pack of this repository's registry holds" + for _, name := range []string{"refs/heads/planted", "refs/tags/planted", "refs/notes/planted"} { + t.Run(name, func(t *testing.T) { + report, remote := rawReceivePack(t, node, repo, token, false, emptyPack, rawPushCommand{ref.ZeroSHA, cacheOnly, name}) + assertNoCrashText(t, remote) + if !strings.HasPrefix(report[name], "ng") || !strings.Contains(remote, refused) { + t.Fatalf("%s at a commit that only the cache holds was not refused with %q: report %v\nremote:\n%s", name, refused, report, remote) + } + if got, ok := pgRefs(t, repo)[name]; ok { + t.Fatalf("%s = %s after a refused push, want no ref", name, got) + } + }) + } + + // A commit that a registered pack holds still works, with the same empty pack. + t.Run("a registered commit", func(t *testing.T) { + const name = "refs/heads/copy" + report, remote := rawReceivePack(t, node, repo, token, false, emptyPack, rawPushCommand{ref.ZeroSHA, c1, name}) + if report[name] != "ok" || pgRefs(t, repo)[name] != c1 { + t.Fatalf("a branch at a registered commit: report %v, ref %s\nremote:\n%s", report, pgRefs(t, repo)[name], remote) + } + }) +} diff --git a/internal/gitread/gitread.go b/internal/gitread/gitread.go index da9671f5..b407f253 100644 --- a/internal/gitread/gitread.go +++ b/internal/gitread/gitread.go @@ -272,3 +272,30 @@ func resolveCommitObject(ctx context.Context, repoPath, cand, id string) (string } return "", fmt.Errorf("git cannot read commit or tag chain %s", id) } + +// RegisteredKinds returns the type of each id as the registered packs show it: +// commit, tree, blob, tag, or "missing" when no pack that the registry lists +// for repo holds the object. It reads a view of the cache that links only those +// packs (githttp.RegisteredPackView), so an object that only the node's cache +// holds reads as missing. ResolveRef and the other reads here consult the +// cache, which can hold more than the registry. A write that stores an object id +// in a ref must ask this, because every other node hydrates from the packs. +// A registered pack that the cache lacks is downloaded first. +func RegisteredKinds(ctx context.Context, st ref.RefStore, bl blob.BlobStore, repoPath, repo string, ids []string) (map[string]string, error) { + if len(ids) == 0 { + return map[string]string{}, nil + } + view, err := githttp.NewRegisteredPackView(ctx, st, repoPath, repo, func(ctx context.Context) error { + return githttp.HydratePacks(ctx, st, bl, repoPath, repo) + }) + if err != nil { + return nil, err + } + defer view.Close() + out, err := Run(ctx, Cmd{Class: Light, Dir: repoPath, Env: view.Env(os.Environ()), Stdin: strings.NewReader(strings.Join(ids, "\n") + "\n"), + Args: []string{"cat-file", "--batch-check=%(objecttype)"}}) + if err != nil { + return nil, err + } + return githttp.ParseObjectKinds(ids, out) +}