From e1d4a797802da859205fc67494aa719ff23e68a3 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 16:41:20 +0000 Subject: [PATCH 1/2] fix(githttp): refuse a ref at an object that no registered pack holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node's cache repository can hold an object that no registered pack holds (a cache that was copied or edited out of band, or a registry that was rolled back). git reads such an object, so three entry points stored a ref at it: POST branches/create and POST tags resolved the base or the target in the cache, and a push update with an empty pack passed git's connectivity check. Every other node hydrates from the registered packs alone, so each such ref wedged the repository. The two REST routes now answer 404 for a commit that no registered pack holds, and the update hook refuses the update, both by object state against a view of the cache that links only the packs that the registry lists. Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-sonnet-5-5` • Thinking: `xhigh`_ Change-Id: Ia072c3e1bd28cbd9b2a4441250dae1d810a2d150 --- docs/reference/http-api.md | 9 ++ internal/api/handlers_write_refs.go | 29 +++- internal/api/registry_kinds_test.go | 72 +++++++++ internal/api/registry_target_test.go | 106 ++++++++++++++ internal/githttp/hook.go | 60 ++++++++ internal/githttp/hookcanary_test.go | 11 ++ internal/githttp/hookrefusal_test.go | 1 + internal/githttp/registryview.go | 138 ++++++++++++++++++ internal/githttp/registryview_test.go | 31 ++++ internal/githttp/s3wal_opcensus_test.go | 5 +- .../githttp/unregistered_target_e2e_test.go | 64 ++++++++ internal/gitread/gitread.go | 27 ++++ 12 files changed, 549 insertions(+), 4 deletions(-) create mode 100644 internal/api/registry_kinds_test.go create mode 100644 internal/api/registry_target_test.go create mode 100644 internal/githttp/registryview.go create mode 100644 internal/githttp/registryview_test.go create mode 100644 internal/githttp/unregistered_target_e2e_test.go diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index a8e5bdd56..e14c719b0 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -683,6 +683,15 @@ old behavior, read `default_branch` from `GET /repo` and send it as `base_ref`. This follows the public OpenAPI/SDK contract, not demonstrated upstream cluster behavior ([contract audit](../explanation/wire-compatibility.md#server-allocated-branch-contract)). +#### Which commit the base and the tag target can name + +The commit that `base_ref`, `base_branch` or `preferred_base` of `POST /repos/branches/create`, or +`ref` of `POST /repos/tags`, names must be in a pack that the repository's pack registry lists. +The server checks the registered packs, not the node's cache repository, because every other node +(and this node after it loses its cache) hydrates from the registered packs alone and cannot hold a +ref at any other object. A commit that only a cache holds answers `404`, like a commit that does +not exist, and a `base_ref` that names no usable commit still falls back to `preferred_base`. + #### Guarded branch create and move `expected_target_sha` on `POST /repos/branches/create` (both path styles) guards the diff --git a/internal/api/handlers_write_refs.go b/internal/api/handlers_write_refs.go index f04e92f58..5d96bf634 100644 --- a/internal/api/handlers_write_refs.go +++ b/internal/api/handlers_write_refs.go @@ -386,6 +386,29 @@ func wrefValidateCreateBranch(req *wrefCreateBranchRequest) (target string, allo return target, allocate, guard, errs.err() } +// wrefResolveHeld is gitread.ResolveRef for a write that stores the commit in a +// ref. The commit must also be in a pack that the registry lists for the +// repository: the node's cache can hold objects that no registered pack holds +// (a cache that was copied or edited out of band, or a registry that was rolled +// back), and a ref at such an object wedges every other node and this node +// after its cache is gone, because they hydrate from the registered packs +// alone. A commit that no registered pack holds answers like one that does not +// exist. +func (s *server) wrefResolveHeld(ctx context.Context, repoID, repoPath, spec string, ephemeral bool) (string, error) { + sha, err := gitread.ResolveRef(ctx, repoPath, spec, ephemeral) + if err != nil { + return "", err + } + kinds, err := gitread.RegisteredKinds(ctx, s.refStore, s.blobStore, repoPath, repoID, []string{sha}) + if err != nil { + return "", fmt.Errorf("check that a registered pack holds %s: %w", sha, err) + } + if kinds[sha] != "commit" { + return "", gitread.ErrNotFound + } + return sha, nil +} + // handleCreateBranch resolves the base to a commit (branch, tag — peeled — // or hex sha via gitread.ResolveRef; preferred_base is the fallback when // the primary base is absent or names no ref, and a base whose commit git @@ -436,7 +459,7 @@ func (s *server) handleCreateBranch(w http.ResponseWriter, r *http.Request) { ctx := r.Context() baseSHA := "" if base != "" { - sha, err := gitread.ResolveRef(ctx, repoPath, base, req.BaseIsEphemeral) + sha, err := s.wrefResolveHeld(ctx, repo.RepoID, repoPath, base, req.BaseIsEphemeral) switch { case errors.Is(err, gitread.ErrNotFound): // Fall through to preferred_base below (when given). An @@ -449,7 +472,7 @@ func (s *server) handleCreateBranch(w http.ResponseWriter, r *http.Request) { } } if baseSHA == "" && req.PreferredBase != "" { - sha, err := gitread.ResolveRef(ctx, repoPath, req.PreferredBase, req.BaseIsEphemeral) + sha, err := s.wrefResolveHeld(ctx, repo.RepoID, repoPath, req.PreferredBase, req.BaseIsEphemeral) switch { case errors.Is(err, gitread.ErrNotFound): // Report below with the primary-base message. @@ -752,7 +775,7 @@ func (s *server) handleCreateTag(w http.ResponseWriter, r *http.Request) { if !ok { return } - sha, err := gitread.ResolveRef(r.Context(), repoPath, target, false) + sha, err := s.wrefResolveHeld(r.Context(), repo.RepoID, repoPath, target, false) switch { case errors.Is(err, gitread.ErrNotFound): writeProblem(w, http.StatusNotFound, codeRefNotFound, "ref does not resolve to a commit") diff --git a/internal/api/registry_kinds_test.go b/internal/api/registry_kinds_test.go new file mode 100644 index 000000000..95a9cd62e --- /dev/null +++ b/internal/api/registry_kinds_test.go @@ -0,0 +1,72 @@ +package api + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/coder/openstorage/internal/githttp" + "github.com/coder/openstorage/internal/gitread" +) + +// RegisteredKinds answers from the registered packs alone: a commit that only +// the cache holds reads as missing, and a registered pack that the cache lacks +// (a fold replaced the packs since the cache was hydrated) is downloaded first, +// so that a healthy object never reads as missing. +func TestRegisteredKindsReadsTheRegisteredPacksOnly(t *testing.T) { + runBackends(t, func(t *testing.T, e *testEnv) { + fx := wrefSeedFixture(t, e) + ctx := context.Background() + repoPath, err := gitread.Materialize(ctx, e.refStore, e.blobStore, e.cacheDir, fx.repoID, "main") + if err != nil { + t.Fatal(err) + } + planted := plantCacheOnlyCommit(t, repoPath, fx.c2) + kinds, err := gitread.RegisteredKinds(ctx, e.refStore, e.blobStore, repoPath, fx.repoID, []string{fx.c1, fx.c2, fx.tagObj, planted}) + if err != nil { + t.Fatal(err) + } + want := map[string]string{fx.c1: "commit", fx.c2: "commit", fx.tagObj: "tag", planted: "missing"} + for id, k := range want { + if kinds[id] != k { + t.Fatalf("kind of %s = %q, want %q (all: %v)", id, kinds[id], k, kinds) + } + } + + // The cache loses its pack files: the registry still lists the packs. + packs, err := filepath.Glob(filepath.Join(repoPath, "objects", "pack", "pack-*")) + if err != nil || len(packs) == 0 { + t.Fatalf("pack files in the cache: %v, %v", packs, err) + } + for _, p := range packs { + if err := os.Remove(p); err != nil { + t.Fatal(err) + } + } + kinds, err = gitread.RegisteredKinds(ctx, e.refStore, e.blobStore, repoPath, fx.repoID, []string{fx.c2}) + if err != nil || kinds[fx.c2] != "commit" { + t.Fatalf("after the cache lost its packs: %v, %v", kinds, err) + } + }) +} + +// With no way to fetch a pack that the cache lacks, the view refuses, so that +// a caller never reads "missing" for an object that a registered pack holds. +func TestRegisteredPackViewRefusesWhenAPackIsMissingAndCannotBeFetched(t *testing.T) { + e := testEnvironment(t, "sqlite") + fx := wrefSeedFixture(t, e) + repoPath := filepath.Join(t.TempDir(), "empty.git") + if err := os.MkdirAll(filepath.Join(repoPath, "objects", "pack"), 0o755); err != nil { + t.Fatal(err) + } + if _, err := githttp.NewRegisteredPackView(context.Background(), e.refStore, repoPath, fx.repoID, nil); !errors.Is(err, githttp.ErrPackMissing) { + t.Fatalf("NewRegisteredPackView with no heal = %v, want ErrPackMissing", err) + } + calls := 0 + _, err := githttp.NewRegisteredPackView(context.Background(), e.refStore, repoPath, fx.repoID, func(context.Context) error { calls++; return nil }) + if !errors.Is(err, githttp.ErrPackMissing) || calls != 1 { + t.Fatalf("NewRegisteredPackView with a heal that fetches nothing = %v after %d heals, want ErrPackMissing after 1", err, calls) + } +} diff --git a/internal/api/registry_target_test.go b/internal/api/registry_target_test.go new file mode 100644 index 000000000..7ff7e7d80 --- /dev/null +++ b/internal/api/registry_target_test.go @@ -0,0 +1,106 @@ +package api + +// A ref write must not store a commit that only the node's cache holds. The +// cache can hold objects that no registered pack holds (a cache that was copied +// or edited out of band, or a registry that was rolled back), and a ref at such +// an object wedges every other node, and this node after its cache is gone, +// because they hydrate from the registered packs alone. + +import ( + "context" + "net/http" + "os" + "os/exec" + "strings" + "testing" + + "github.com/coder/openstorage/internal/gitread" +) + +// plantCacheOnlyCommit writes a commit (a child of parent, with a new blob) as +// loose objects into the cache repository at repoPath only: no pack holds it. +func plantCacheOnlyCommit(t *testing.T, repoPath, parent string) string { + t.Helper() + run := func(stdin string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repoPath + for _, kv := range os.Environ() { + if !strings.HasPrefix(kv, "GIT_") { + cmd.Env = append(cmd.Env, kv) + } + } + cmd.Env = append(cmd.Env, "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null", + "GIT_AUTHOR_NAME=p", "GIT_AUTHOR_EMAIL=p@x.invalid", "GIT_COMMITTER_NAME=p", "GIT_COMMITTER_EMAIL=p@x.invalid") + cmd.Stdin = strings.NewReader(stdin) + out, err := cmd.Output() + if err != nil { + t.Fatalf("git %v: %v", args, err) + } + return strings.TrimSpace(string(out)) + } + blob := run("cache only\n", "hash-object", "-w", "--stdin") + tree := run("100644 blob "+blob+"\tcacheonly.txt\n", "mktree") + return run("", "commit-tree", "-p", parent, "-m", "cache only", tree) +} + +func TestRefWritesRefuseACommitThatOnlyTheCacheHolds(t *testing.T) { + type shape struct { + name, path string + body func(sha string) map[string]any + ref string // the ref that the request would create + } + shapes := []shape{ + {"branch base_ref", "/api/v1/repos/branches/create", func(sha string) map[string]any { + return map[string]any{"target_branch": "co", "base_ref": sha} + }, "refs/heads/co"}, + {"branch preferred_base", "/api/v1/repos/branches/create", func(sha string) map[string]any { + return map[string]any{"target_branch": "co", "base_ref": "nosuch", "preferred_base": sha} + }, "refs/heads/co"}, + {"branch force", "/api/v1/repos/branches/create", func(sha string) map[string]any { + return map[string]any{"target_branch": "feature", "base_ref": sha, "force": true} + }, "refs/heads/feature"}, + {"ephemeral branch", "/api/v1/repos/branches/create", func(sha string) map[string]any { + return map[string]any{"target_branch": "co", "base_ref": sha, "base_is_ephemeral": true, "target_is_ephemeral": true} + }, "refs/namespaces/ephemeral/refs/heads/co"}, + {"tag", "/api/v1/repos/tags", func(sha string) map[string]any { + return map[string]any{"name": "co", "ref": sha} + }, "refs/tags/co"}, + } + runBackends(t, func(t *testing.T, e *testEnv) { + for _, tc := range shapes { + t.Run(tc.name, func(t *testing.T) { + fx := wrefSeedFixture(t, e) + repoPath, err := gitread.Materialize(context.Background(), e.refStore, e.blobStore, e.cacheDir, fx.repoID, "main") + if err != nil { + t.Fatal(err) + } + planted := plantCacheOnlyCommit(t, repoPath, fx.c2) + srv := newWrefServer(t, e) + before := wrefRefs(t, e, fx.repoID) + + resp, body := wrefDo(t, srv, http.MethodPost, tc.path, fx.token, tc.body(planted)) + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("a commit that only the cache holds: status %d, want 404, body %v", resp.StatusCode, body) + } + after := wrefRefs(t, e, fx.repoID) + if len(after) != len(before) || after[tc.ref] != before[tc.ref] { + t.Fatalf("refs changed after the refused request: before %v, after %v", before, after) + } + // Every other node hydrates from the registered packs alone. + if _, err := gitread.Materialize(context.Background(), e.refStore, e.blobStore, t.TempDir(), fx.repoID, "main"); err != nil { + t.Fatalf("a fresh node cannot hydrate the repository: %v", err) + } + + // A commit that a registered pack holds still works. + resp, body = wrefDo(t, srv, http.MethodPost, tc.path, fx.token, tc.body(fx.c1)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("a commit that a pack holds: status %d, want 200, body %v", resp.StatusCode, body) + } + if got := wrefRefs(t, e, fx.repoID)[tc.ref]; got != fx.c1 { + t.Fatalf("%s = %q, want %s", tc.ref, got, fx.c1) + } + }) + } + }) +} diff --git a/internal/githttp/hook.go b/internal/githttp/hook.go index adc2ffba8..e96a3f4bd 100644 --- a/internal/githttp/hook.go +++ b/internal/githttp/hook.go @@ -1840,6 +1840,21 @@ func updateHook(ctx context.Context, args []string) (retErr error) { return err } } + // The new value must be in a pack that the registry lists. git has already + // checked that the object is readable, but it reads the node's cache, which + // can hold objects that no registered pack holds (a cache that was copied or + // edited out of band, or a registry that was rolled back). A ref at such an + // object wedges every node that hydrates from the registered packs. A push + // that carries its objects is safe: pre-receive registered its pack before + // this hook runs, and git moved the pack into the cache. The case that this + // closes is an update to an object that no pack of this push carries. + // Checked on the raw store, before the CAS, and denied per ref like a policy + // denial. Deletions have no new value. + if new != ref.ZeroSHA { + if err := unregisteredTargetCheck(ctx, st, repo, refName, new); err != nil { + return err + } + } var inner ref.RefStore = st if testMode() { inner = injectCASDrop(inner) @@ -2917,6 +2932,51 @@ func objectExists(sha string) bool { panic(fmt.Sprintf("githttp assertion failed: cat-file -e %s: %v", sha, err)) } +// unregisteredTargetCheck is denyUnregisteredTarget. Tests that run the update +// hook on an object id that they invented, to reach a later step, replace it. +var unregisteredTargetCheck = denyUnregisteredTarget + +// denyUnregisteredTarget refuses the update of refName to new when no pack that +// the registry lists for repo holds the object. It reads the cache repository +// through a RegisteredPackView, so only the registered packs answer. A registered +// pack that the cache lacks (a fold replaced the packs after the cache was +// hydrated) is downloaded first. An object that is no commit is left to the +// other checks: only "missing" is refused here. +func denyUnregisteredTarget(ctx context.Context, st ref.RefStore, repo, refName, new string) error { + repoPath, err := os.Getwd() // git runs hooks in the repository + if err != nil { + return fmt.Errorf("update: find the repository: %w", err) + } + view, err := NewRegisteredPackView(ctx, st, repoPath, repo, func(ctx context.Context) error { + bl, err := blob.Open(ctx, mustEnv(envHookBlobURL...)) + if err != nil { + return err + } + defer bl.Close() + return HydratePacks(ctx, st, bl, repoPath, repo) + }) + if err != nil { + return fmt.Errorf("update: read the registered packs: %w", err) + } + defer view.Close() + var out strings.Builder + cmd := exec.CommandContext(ctx, "git", "cat-file", "--batch-check=%(objecttype)") + cmd.Env = gitlimit.NoReplaceObjects(view.Env(os.Environ())) + cmd.Stdin = strings.NewReader(new + "\n") + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return fmt.Errorf("update: read the object type of %s in the registered packs: %w", new, err) + } + kinds, err := ParseObjectKinds([]string{new}, out.String()) + if err != nil { + return fmt.Errorf("update: %w", err) + } + if kinds[new] == "missing" { + return refusef("update: %s is at %s, which no pack of this repository's registry holds", refName, new) + } + return nil +} + // objectInMainStore reports whether sha is resolvable WITHOUT the // quarantine. git runs pre-receive with GIT_OBJECT_DIRECTORY pointed at the // quarantine and the main store as an alternate (githooks(5)); stripping diff --git a/internal/githttp/hookcanary_test.go b/internal/githttp/hookcanary_test.go index c8d929062..028e1e2fb 100644 --- a/internal/githttp/hookcanary_test.go +++ b/internal/githttp/hookcanary_test.go @@ -65,6 +65,16 @@ func canarySQLExec(t *testing.T, dbURL string, stmts ...string) { } } +// skipRegisteredTargetCheck lets a test run the update hook on an object id +// that it invented: the check that a registered pack holds the new value would +// refuse it before the step that the test wants to reach. +func skipRegisteredTargetCheck(t *testing.T) { + t.Helper() + prev := unregisteredTargetCheck + unregisteredTargetCheck = func(context.Context, ref.RefStore, string, string, string) error { return nil } + t.Cleanup(func() { unregisteredTargetCheck = prev }) +} + // canaryTable is a unique name for a table that does not exist. func canaryTable() string { return "canary_" + strings.ToLower(rand.Text()[:8]) } @@ -118,6 +128,7 @@ func TestUpdateHookHidesStepErrors(t *testing.T) { t.Run("WrapHookRefStore error", func(t *testing.T) { canary := canaryTable() records := hookEnv(t, canarySQLite(t)) + skipRegisteredTargetCheck(t) prev := WrapHookRefStore t.Cleanup(func() { WrapHookRefStore = prev }) WrapHookRefStore = func(context.Context, ref.RefStore) (ref.RefStore, error) { diff --git a/internal/githttp/hookrefusal_test.go b/internal/githttp/hookrefusal_test.go index dcb774469..741046c0c 100644 --- a/internal/githttp/hookrefusal_test.go +++ b/internal/githttp/hookrefusal_test.go @@ -154,6 +154,7 @@ func TestUpdateHookHidesStoreErrors(t *testing.T) { st.Close() c := pgCanary() records := hookEnv(t, dbURL) + skipRegisteredTargetCheck(t) prev := WrapHookRefStore t.Cleanup(func() { WrapHookRefStore = prev }) WrapHookRefStore = func(_ context.Context, st ref.RefStore) (ref.RefStore, error) { diff --git a/internal/githttp/registryview.go b/internal/githttp/registryview.go new file mode 100644 index 000000000..ee1ec7075 --- /dev/null +++ b/internal/githttp/registryview.go @@ -0,0 +1,138 @@ +package githttp + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/coder/openstorage/internal/ref" +) + +// A node's cache repository can hold an object that no registered pack holds: +// a cache that was copied, restored or edited out of band, or a registry that +// was rolled back. git reads such an object without complaint, so "git can +// read it" proves nothing about the repository's durable state. Every other +// node, and this node after its cache is gone, hydrates from the registered +// packs alone and cannot write a ref at that object. +// +// RegisteredPackView is the repository as those nodes see it: an object +// directory that holds exactly the packs that the registry lists. A git child +// that runs with GIT_OBJECT_DIRECTORY set to it, and with no alternates, reads +// an object only when a registered pack holds it. + +// RegisteredPackView is a temporary object directory that links the pack files +// of the repository's cache that the registry lists, and nothing else. +type RegisteredPackView struct { + dir string + repoPath string +} + +// NewRegisteredPackView lists the packs that st has for repo and links their +// files from repoPath's object directory. A registered pack that is not in the +// cache (a fold replaced the packs since the cache was last hydrated) is +// fetched by heal, once, and the listing is read again. A nil heal, or a pack +// that is still missing after it, is an error that wraps ErrPackMissing: the +// view would lack objects that the registry holds, and a caller that read +// "missing" from it would refuse a ref that is healthy. +func NewRegisteredPackView(ctx context.Context, st ref.RefStore, repoPath, repo string, heal func(context.Context) error) (*RegisteredPackView, error) { + packDir := filepath.Join(repoPath, "objects", "pack") + var packs []ref.Pack + for attempt := 0; ; attempt++ { + var err error + if packs, err = st.ListPacks(ctx, repo); err != nil { + return nil, fmt.Errorf("list the packs of %s: %w", repo, err) + } + missing := "" + for _, p := range packs { + if !packFilesExist(packDir, p.ID) { + missing = p.ID + break + } + } + if missing == "" { + break + } + if heal == nil || attempt > 0 { + return nil, fmt.Errorf("%w: the cache of %s lacks the registered pack %s", ErrPackMissing, repo, missing) + } + if err := heal(ctx); err != nil { + return nil, fmt.Errorf("fetch the registered packs of %s: %w", repo, err) + } + } + dir, err := os.MkdirTemp("", "openstorage-registry-view-") + if err != nil { + return nil, err + } + v := &RegisteredPackView{dir: dir, repoPath: repoPath} + viewPacks := filepath.Join(dir, "pack") + if err := os.Mkdir(viewPacks, 0o700); err != nil { + v.Close() + return nil, err + } + for _, p := range packs { + for _, ext := range []string{".pack", ".idx"} { + name := "pack-" + p.ID + ext + if err := os.Symlink(filepath.Join(packDir, name), filepath.Join(viewPacks, name)); err != nil { + v.Close() + return nil, err + } + } + } + return v, nil +} + +func packFilesExist(packDir, id string) bool { + for _, ext := range []string{".pack", ".idx"} { + if _, err := os.Stat(filepath.Join(packDir, "pack-"+id+ext)); err != nil { + return false + } + } + return true +} + +// Close removes the view. The pack files that it linked stay. +func (v *RegisteredPackView) Close() { + if v != nil && v.dir != "" { + _ = os.RemoveAll(v.dir) + } +} + +// Env returns base with the variables that point git at the cache repository +// and at the view, and none of git's other repository variables. +func (v *RegisteredPackView) Env(base []string) []string { + out := make([]string, 0, len(base)+2) + for _, kv := range base { + name, _, _ := strings.Cut(kv, "=") + switch name { + case "GIT_DIR", "GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_QUARANTINE_PATH": + default: + out = append(out, kv) + } + } + return append(out, "GIT_DIR="+v.repoPath, "GIT_OBJECT_DIRECTORY="+v.dir) +} + +// ParseObjectKinds reads the output of `git cat-file +// --batch-check=%(objecttype)` for ids: one line for each id, the object's +// type or " missing". +func ParseObjectKinds(ids []string, out string) (map[string]string, error) { + lines := strings.Split(strings.TrimSuffix(out, "\n"), "\n") + if len(lines) != len(ids) { + return nil, fmt.Errorf("cat-file --batch-check answered %d lines for %d ids", len(lines), len(ids)) + } + kinds := make(map[string]string, len(ids)) + for i, l := range lines { + switch l { + case "commit", "tree", "blob", "tag": + kinds[ids[i]] = l + case ids[i] + " missing": + kinds[ids[i]] = "missing" + default: + return nil, errors.New("cat-file --batch-check answered an unknown line for " + ids[i]) + } + } + return kinds, nil +} diff --git a/internal/githttp/registryview_test.go b/internal/githttp/registryview_test.go new file mode 100644 index 000000000..293c8b8a3 --- /dev/null +++ b/internal/githttp/registryview_test.go @@ -0,0 +1,31 @@ +package githttp + +import ( + "slices" + "strings" + "testing" +) + +func TestParseObjectKinds(t *testing.T) { + a, b, c := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40) + got, err := ParseObjectKinds([]string{a, b, c}, "commit\n"+b+" missing\ntag\n") + if err != nil || got[a] != "commit" || got[b] != "missing" || got[c] != "tag" { + t.Fatalf("ParseObjectKinds = %v, %v", got, err) + } + for _, bad := range []string{"commit\n", "commit\nbogus\ntag\n", "commit\n" + c + " missing\ntag\n"} { + if _, err := ParseObjectKinds([]string{a, b, c}, bad); err == nil { + t.Fatalf("ParseObjectKinds(%q) = nil error, want a refusal", bad) + } + } +} + +// The view points git at the cache repository and at the view directory, and at +// no alternate or quarantine that the caller's environment names. +func TestRegisteredPackViewEnvReplacesTheObjectVariables(t *testing.T) { + v := &RegisteredPackView{dir: "/view", repoPath: "/cache/r.git"} + env := v.Env([]string{"PATH=/bin", "GIT_DIR=.", "GIT_OBJECT_DIRECTORY=/q", "GIT_ALTERNATE_OBJECT_DIRECTORIES=/main", "GIT_QUARANTINE_PATH=/q", "GIT_COMMON_DIR=/c", "HOME=/h"}) + want := []string{"PATH=/bin", "HOME=/h", "GIT_DIR=/cache/r.git", "GIT_OBJECT_DIRECTORY=/view"} + if !slices.Equal(env, want) { + t.Fatalf("Env = %v, want %v", env, want) + } +} diff --git a/internal/githttp/s3wal_opcensus_test.go b/internal/githttp/s3wal_opcensus_test.go index 6eb8f274e..1d714e18e 100644 --- a/internal/githttp/s3wal_opcensus_test.go +++ b/internal/githttp/s3wal_opcensus_test.go @@ -209,7 +209,10 @@ func TestS3WAL_PushOpCensus(t *testing.T) { // update hook's format check before its CAS (issue #445): one GET of // _wal/meta.json, so a wrong metadata format refuses the update instead // of dropping its events. The post-CAS reads then revalidate with 304s. - const steadyBudget = 21 + // 22 = the update hook's check that a registered pack holds the new value + // (denyUnregisteredTarget): one listing of the registered packs, a manifest + // read. A delete has no new value, so the delete-only budget is unchanged. + const steadyBudget = 22 if len(steady) > steadyBudget { t.Errorf("steady-state push cost %d S3 round trips, budget is %d:\n%s", len(steady), steadyBudget, censusTable(steady)) diff --git a/internal/githttp/unregistered_target_e2e_test.go b/internal/githttp/unregistered_target_e2e_test.go new file mode 100644 index 000000000..25c791222 --- /dev/null +++ b/internal/githttp/unregistered_target_e2e_test.go @@ -0,0 +1,64 @@ +package githttp + +// A node's cache repository can hold an object that no registered pack holds (a +// cache that was copied or edited out of band, or a registry that was rolled +// back). git reads such an object, so its connectivity check lets a push update +// a ref to it, with an empty pack that carries nothing. Every other node +// hydrates from the registered packs alone and cannot write a ref at that +// object, so the ref would wedge the repository. The update hook refuses it. + +import ( + "os/exec" + "strings" + "testing" + + "github.com/coder/openstorage/internal/ref" +) + +func TestUpdateHookRefusesATargetThatNoRegisteredPackHolds(t *testing.T) { + cache := t.TempDir() + node, err := startNode(cache) + if err != nil { + t.Fatalf("start node: %v", err) + } + t.Cleanup(node.stop) + repo := registerRepo(t, "t-unregistered-target") + work := initWorkRepo(t) + token := mintTestToken(t, repo, "git:read,git:write", "") + mustGit(t, work, "push", remoteURL(node, repo, token), "HEAD:refs/heads/main") + c1 := revParse(t, work, "HEAD") + tree := revParse(t, work, c1+"^{tree}") + + // The node's cache holds a commit that no registered pack holds. Its tree + // and parent are in a registered pack, so git's connectivity check passes. + cacheOnly := plantCommit(t, RepoPath(cache, repo), tree, c1) + cmd := exec.Command("git", "pack-objects", "--stdout", "-q") + cmd.Dir = work + emptyPack, err := cmd.Output() // no object ids on stdin: a pack with no objects + if err != nil { + t.Fatalf("git pack-objects of nothing: %v", err) + } + + const refused = "no pack of this repository's registry holds" + for _, name := range []string{"refs/heads/planted", "refs/tags/planted", "refs/notes/planted"} { + t.Run(name, func(t *testing.T) { + report, remote := rawReceivePack(t, node, repo, token, false, emptyPack, rawPushCommand{ref.ZeroSHA, cacheOnly, name}) + assertNoCrashText(t, remote) + if !strings.HasPrefix(report[name], "ng") || !strings.Contains(remote, refused) { + t.Fatalf("%s at a commit that only the cache holds was not refused with %q: report %v\nremote:\n%s", name, refused, report, remote) + } + if got, ok := pgRefs(t, repo)[name]; ok { + t.Fatalf("%s = %s after a refused push, want no ref", name, got) + } + }) + } + + // A commit that a registered pack holds still works, with the same empty pack. + t.Run("a registered commit", func(t *testing.T) { + const name = "refs/heads/copy" + report, remote := rawReceivePack(t, node, repo, token, false, emptyPack, rawPushCommand{ref.ZeroSHA, c1, name}) + if report[name] != "ok" || pgRefs(t, repo)[name] != c1 { + t.Fatalf("a branch at a registered commit: report %v, ref %s\nremote:\n%s", report, pgRefs(t, repo)[name], remote) + } + }) +} diff --git a/internal/gitread/gitread.go b/internal/gitread/gitread.go index da9671f51..b407f2537 100644 --- a/internal/gitread/gitread.go +++ b/internal/gitread/gitread.go @@ -272,3 +272,30 @@ func resolveCommitObject(ctx context.Context, repoPath, cand, id string) (string } return "", fmt.Errorf("git cannot read commit or tag chain %s", id) } + +// RegisteredKinds returns the type of each id as the registered packs show it: +// commit, tree, blob, tag, or "missing" when no pack that the registry lists +// for repo holds the object. It reads a view of the cache that links only those +// packs (githttp.RegisteredPackView), so an object that only the node's cache +// holds reads as missing. ResolveRef and the other reads here consult the +// cache, which can hold more than the registry. A write that stores an object id +// in a ref must ask this, because every other node hydrates from the packs. +// A registered pack that the cache lacks is downloaded first. +func RegisteredKinds(ctx context.Context, st ref.RefStore, bl blob.BlobStore, repoPath, repo string, ids []string) (map[string]string, error) { + if len(ids) == 0 { + return map[string]string{}, nil + } + view, err := githttp.NewRegisteredPackView(ctx, st, repoPath, repo, func(ctx context.Context) error { + return githttp.HydratePacks(ctx, st, bl, repoPath, repo) + }) + if err != nil { + return nil, err + } + defer view.Close() + out, err := Run(ctx, Cmd{Class: Light, Dir: repoPath, Env: view.Env(os.Environ()), Stdin: strings.NewReader(strings.Join(ids, "\n") + "\n"), + Args: []string{"cat-file", "--batch-check=%(objecttype)"}}) + if err != nil { + return nil, err + } + return githttp.ParseObjectKinds(ids, out) +} From cac01d4757feedff4a408faa45047acbde625741 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 17:38:03 +0000 Subject: [PATCH 2/2] chore(verify): record lane-schedule fingerprints for the registered-target check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-sonnet-5-5` • Thinking: `xhigh`_ Change-Id: I89f3a475d656e81fd175b9b8f2c4c980c2409bfd --- .agents/skills/verify-lane-schedule/SKILL.md | 2 +- .agents/skills/verify-lane-schedule/model/fingerprints.txt | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.agents/skills/verify-lane-schedule/SKILL.md b/.agents/skills/verify-lane-schedule/SKILL.md index 6551defcd..2833d3e92 100644 --- a/.agents/skills/verify-lane-schedule/SKILL.md +++ b/.agents/skills/verify-lane-schedule/SKILL.md @@ -226,7 +226,7 @@ With every fact TRUE in the code, setting each fact FALSE in the model alone fai ## Coverage record -Sync commit: `0f5979dd0589da6224230f74e9bc7d15143ef5d1` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). +Sync commit: `e1d4a797802da859205fc67494aa719ff23e68a3` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). Every declaration in these files is covered: diff --git a/.agents/skills/verify-lane-schedule/model/fingerprints.txt b/.agents/skills/verify-lane-schedule/model/fingerprints.txt index 42532cfbb..0564895b0 100644 --- a/.agents/skills/verify-lane-schedule/model/fingerprints.txt +++ b/.agents/skills/verify-lane-schedule/model/fingerprints.txt @@ -181,6 +181,7 @@ internal/githttp/hook.go:const_reservationDenied f72c8a7d40a9207271da47cf160c52e internal/githttp/hook.go:const_reservationJIT d051ee28340ba848f8282f31689c8bbdde129b4970df2be75292d3b4486c429c internal/githttp/hook.go:const_reservationNeedsVerify b8ec7eeca735e84bb9aa7aa596b7961d17778741956392c03d71116285ff5730 internal/githttp/hook.go:denyAuthorityDFConflict 3ad4a9b7cd317dbae7ebf368256b119a33774f58734c26fef36d47545f50ba5b +internal/githttp/hook.go:denyUnregisteredTarget a07930a318a3bd3488190e6869d6ba1d7589354257c8b832d66e43b733ad665c internal/githttp/hook.go:enforceRefPolicy 2cd2cf6aea243221e8fe27280a6333695d4455a0304e7f536f7e8729dbb17a4d internal/githttp/hook.go:enforceSyncClaimFence efb7b07e54bbb4008ac0cf53778c2ba690ad936051c103421aaf308b52e1cdf9 internal/githttp/hook.go:evalRefPolicy b232fd0a747ced157cec1bc4588e53bc13c54a39ff829f45108ae62c9601b5c8 @@ -248,7 +249,7 @@ internal/githttp/hook.go:type_refValuesProber 0389595e3fb341c7e67784d27674d922dd internal/githttp/hook.go:type_reservationClass 48d346b5a457165c93ed2a9aca43bcd39438f05201427fa2351d8b3a4066ff29 internal/githttp/hook.go:type_sigVerifier afbcc825325e58413e4a3646073bab61862ebda41b2c8c9f6fefae2ad35a4745 internal/githttp/hook.go:updateBarrierEachPath 53afd6c803fba2472970b4a81e706f4e6c975c84753c0c94714bd52c1fdd0fcf -internal/githttp/hook.go:updateHook 2d7d924d17781fd711f5c8395f2b0e3d4511ef5e12e4643ef8c8716a607f670f +internal/githttp/hook.go:updateHook a502272c8ecb19db2bf09a988ed121d8b7be88f4fff859137bf1714f60e0cb6d internal/githttp/hook.go:var_WrapHookRefStore 2a84a08163beaeaef666d54ff198e62e68b8e70a83fa6f46f9025b30c28ce5c5 internal/githttp/hook.go:var__ 14f3ac5e5fb70c7fe12bcffeaa3f5891a5f61b624d170427d94afa8153dcf5fd internal/githttp/hook.go:var__#2 d54a83bcc2c7fb6caf4b5040f8c28ca780368fb5c2c578f05c5a48f6edb6dd97 @@ -268,6 +269,7 @@ internal/githttp/hook.go:var_pushRepoDirLen 9db02c4285217113dfb520f541a4af97e86a internal/githttp/hook.go:var_refPolicyCache 12635e37a2a1162463fb6f64c3fa934b9e3793394ff7847bc17bf2015a822e50 internal/githttp/hook.go:var_refPolicyParses e1a83d07f4b25c69151a945c0c6abb1110dd5710769f8c3c8296824f19c1d01b internal/githttp/hook.go:var_refTxFlushAt f6baf6b2e0ccd94a65e622a30218ecf8e574c22c94141b0b99cc4fb2e01e55aa +internal/githttp/hook.go:var_unregisteredTargetCheck c3c0d603df53454230d740514760d74554900d1fb3fbc2d052036fed07e03802 internal/githttp/hook.go:verifySig 51a7f42302b43ac7155d125dd99bbaf3abd305fb30512c5357230bb333414778 internal/githttp/hook.go:verifySigDenied 478714e01ee647624c46ac6f5b5aa04c2c7bac29c61f3cee57bdf7bdc0142a56 internal/githttp/hydrate.go:BaseRepo 20aab83202d8a04a36232730cb5eb3f34f38261ec029f26db7bc44677aeb6a20