Repository navigation
fix(githttp): refuse a ref at an object that no registered pack holds #896
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| package api | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "os" | ||
| "path/filepath" | ||
| "testing" | ||
|
|
||
| "github.com/coder/openstorage/internal/githttp" | ||
| "github.com/coder/openstorage/internal/gitread" | ||
| ) | ||
|
|
||
| // RegisteredKinds answers from the registered packs alone: a commit that only | ||
| // the cache holds reads as missing, and a registered pack that the cache lacks | ||
| // (a fold replaced the packs since the cache was hydrated) is downloaded first, | ||
| // so that a healthy object never reads as missing. | ||
| func TestRegisteredKindsReadsTheRegisteredPacksOnly(t *testing.T) { | ||
| runBackends(t, func(t *testing.T, e *testEnv) { | ||
| fx := wrefSeedFixture(t, e) | ||
| ctx := context.Background() | ||
| repoPath, err := gitread.Materialize(ctx, e.refStore, e.blobStore, e.cacheDir, fx.repoID, "main") | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| planted := plantCacheOnlyCommit(t, repoPath, fx.c2) | ||
| kinds, err := gitread.RegisteredKinds(ctx, e.refStore, e.blobStore, repoPath, fx.repoID, []string{fx.c1, fx.c2, fx.tagObj, planted}) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| want := map[string]string{fx.c1: "commit", fx.c2: "commit", fx.tagObj: "tag", planted: "missing"} | ||
| for id, k := range want { | ||
| if kinds[id] != k { | ||
| t.Fatalf("kind of %s = %q, want %q (all: %v)", id, kinds[id], k, kinds) | ||
| } | ||
| } | ||
|
|
||
| // The cache loses its pack files: the registry still lists the packs. | ||
| packs, err := filepath.Glob(filepath.Join(repoPath, "objects", "pack", "pack-*")) | ||
| if err != nil || len(packs) == 0 { | ||
| t.Fatalf("pack files in the cache: %v, %v", packs, err) | ||
| } | ||
| for _, p := range packs { | ||
| if err := os.Remove(p); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| } | ||
| kinds, err = gitread.RegisteredKinds(ctx, e.refStore, e.blobStore, repoPath, fx.repoID, []string{fx.c2}) | ||
| if err != nil || kinds[fx.c2] != "commit" { | ||
| t.Fatalf("after the cache lost its packs: %v, %v", kinds, err) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| // With no way to fetch a pack that the cache lacks, the view refuses, so that | ||
| // a caller never reads "missing" for an object that a registered pack holds. | ||
| func TestRegisteredPackViewRefusesWhenAPackIsMissingAndCannotBeFetched(t *testing.T) { | ||
| e := testEnvironment(t, "sqlite") | ||
| fx := wrefSeedFixture(t, e) | ||
| repoPath := filepath.Join(t.TempDir(), "empty.git") | ||
| if err := os.MkdirAll(filepath.Join(repoPath, "objects", "pack"), 0o755); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if _, err := githttp.NewRegisteredPackView(context.Background(), e.refStore, repoPath, fx.repoID, nil); !errors.Is(err, githttp.ErrPackMissing) { | ||
| t.Fatalf("NewRegisteredPackView with no heal = %v, want ErrPackMissing", err) | ||
| } | ||
| calls := 0 | ||
| _, err := githttp.NewRegisteredPackView(context.Background(), e.refStore, repoPath, fx.repoID, func(context.Context) error { calls++; return nil }) | ||
| if !errors.Is(err, githttp.ErrPackMissing) || calls != 1 { | ||
| t.Fatalf("NewRegisteredPackView with a heal that fetches nothing = %v after %d heals, want ErrPackMissing after 1", err, calls) | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| package api | ||
|
|
||
| // A ref write must not store a commit that only the node's cache holds. The | ||
| // cache can hold objects that no registered pack holds (a cache that was copied | ||
| // or edited out of band, or a registry that was rolled back), and a ref at such | ||
| // an object wedges every other node, and this node after its cache is gone, | ||
| // because they hydrate from the registered packs alone. | ||
|
|
||
| import ( | ||
| "context" | ||
| "net/http" | ||
| "os" | ||
| "os/exec" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/coder/openstorage/internal/gitread" | ||
| ) | ||
|
|
||
| // plantCacheOnlyCommit writes a commit (a child of parent, with a new blob) as | ||
| // loose objects into the cache repository at repoPath only: no pack holds it. | ||
| func plantCacheOnlyCommit(t *testing.T, repoPath, parent string) string { | ||
| t.Helper() | ||
| run := func(stdin string, args ...string) string { | ||
| t.Helper() | ||
| cmd := exec.Command("git", args...) | ||
| cmd.Dir = repoPath | ||
| for _, kv := range os.Environ() { | ||
| if !strings.HasPrefix(kv, "GIT_") { | ||
| cmd.Env = append(cmd.Env, kv) | ||
| } | ||
| } | ||
| cmd.Env = append(cmd.Env, "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null", | ||
| "GIT_AUTHOR_NAME=p", "GIT_AUTHOR_EMAIL=p@x.invalid", "GIT_COMMITTER_NAME=p", "GIT_COMMITTER_EMAIL=p@x.invalid") | ||
| cmd.Stdin = strings.NewReader(stdin) | ||
| out, err := cmd.Output() | ||
| if err != nil { | ||
| t.Fatalf("git %v: %v", args, err) | ||
| } | ||
| return strings.TrimSpace(string(out)) | ||
| } | ||
| blob := run("cache only\n", "hash-object", "-w", "--stdin") | ||
| tree := run("100644 blob "+blob+"\tcacheonly.txt\n", "mktree") | ||
| return run("", "commit-tree", "-p", parent, "-m", "cache only", tree) | ||
| } | ||
|
|
||
| func TestRefWritesRefuseACommitThatOnlyTheCacheHolds(t *testing.T) { | ||
| type shape struct { | ||
| name, path string | ||
| body func(sha string) map[string]any | ||
| ref string // the ref that the request would create | ||
| } | ||
| shapes := []shape{ | ||
| {"branch base_ref", "/api/v1/repos/branches/create", func(sha string) map[string]any { | ||
| return map[string]any{"target_branch": "co", "base_ref": sha} | ||
| }, "refs/heads/co"}, | ||
| {"branch preferred_base", "/api/v1/repos/branches/create", func(sha string) map[string]any { | ||
| return map[string]any{"target_branch": "co", "base_ref": "nosuch", "preferred_base": sha} | ||
| }, "refs/heads/co"}, | ||
| {"branch force", "/api/v1/repos/branches/create", func(sha string) map[string]any { | ||
| return map[string]any{"target_branch": "feature", "base_ref": sha, "force": true} | ||
| }, "refs/heads/feature"}, | ||
| {"ephemeral branch", "/api/v1/repos/branches/create", func(sha string) map[string]any { | ||
| return map[string]any{"target_branch": "co", "base_ref": sha, "base_is_ephemeral": true, "target_is_ephemeral": true} | ||
| }, "refs/namespaces/ephemeral/refs/heads/co"}, | ||
| {"tag", "/api/v1/repos/tags", func(sha string) map[string]any { | ||
| return map[string]any{"name": "co", "ref": sha} | ||
| }, "refs/tags/co"}, | ||
| } | ||
| runBackends(t, func(t *testing.T, e *testEnv) { | ||
| for _, tc := range shapes { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| fx := wrefSeedFixture(t, e) | ||
| repoPath, err := gitread.Materialize(context.Background(), e.refStore, e.blobStore, e.cacheDir, fx.repoID, "main") | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| planted := plantCacheOnlyCommit(t, repoPath, fx.c2) | ||
| srv := newWrefServer(t, e) | ||
| before := wrefRefs(t, e, fx.repoID) | ||
|
|
||
| resp, body := wrefDo(t, srv, http.MethodPost, tc.path, fx.token, tc.body(planted)) | ||
| if resp.StatusCode != http.StatusNotFound { | ||
| t.Fatalf("a commit that only the cache holds: status %d, want 404, body %v", resp.StatusCode, body) | ||
| } | ||
| after := wrefRefs(t, e, fx.repoID) | ||
| if len(after) != len(before) || after[tc.ref] != before[tc.ref] { | ||
| t.Fatalf("refs changed after the refused request: before %v, after %v", before, after) | ||
| } | ||
| // Every other node hydrates from the registered packs alone. | ||
| if _, err := gitread.Materialize(context.Background(), e.refStore, e.blobStore, t.TempDir(), fx.repoID, "main"); err != nil { | ||
| t.Fatalf("a fresh node cannot hydrate the repository: %v", err) | ||
| } | ||
|
|
||
| // A commit that a registered pack holds still works. | ||
| resp, body = wrefDo(t, srv, http.MethodPost, tc.path, fx.token, tc.body(fx.c1)) | ||
| if resp.StatusCode != http.StatusOK { | ||
| t.Fatalf("a commit that a pack holds: status %d, want 200, body %v", resp.StatusCode, body) | ||
| } | ||
| if got := wrefRefs(t, e, fx.repoID)[tc.ref]; got != fx.c1 { | ||
| t.Fatalf("%s = %q, want %s", tc.ref, got, fx.c1) | ||
| } | ||
| }) | ||
| } | ||
| }) | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1840,6 +1840,21 @@ func updateHook(ctx context.Context, args []string) (retErr error) { | |
| return err | ||
| } | ||
| } | ||
| // The new value must be in a pack that the registry lists. git has already | ||
| // checked that the object is readable, but it reads the node's cache, which | ||
| // can hold objects that no registered pack holds (a cache that was copied or | ||
| // edited out of band, or a registry that was rolled back). A ref at such an | ||
| // object wedges every node that hydrates from the registered packs. A push | ||
| // that carries its objects is safe: pre-receive registered its pack before | ||
| // this hook runs, and git moved the pack into the cache. The case that this | ||
| // closes is an update to an object that no pack of this push carries. | ||
| // Checked on the raw store, before the CAS, and denied per ref like a policy | ||
| // denial. Deletions have no new value. | ||
| if new != ref.ZeroSHA { | ||
| if err := unregisteredTargetCheck(ctx, st, repo, refName, new); err != nil { | ||
| return err | ||
| } | ||
|
Comment on lines
+1853
to
+1856
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🛡️ Codex Security Review
With a Useful? React with 👍 / 👎. |
||
| } | ||
| var inner ref.RefStore = st | ||
| if testMode() { | ||
| inner = injectCASDrop(inner) | ||
|
|
@@ -2917,6 +2932,51 @@ func objectExists(sha string) bool { | |
| panic(fmt.Sprintf("githttp assertion failed: cat-file -e %s: %v", sha, err)) | ||
| } | ||
|
|
||
| // unregisteredTargetCheck is denyUnregisteredTarget. Tests that run the update | ||
| // hook on an object id that they invented, to reach a later step, replace it. | ||
| var unregisteredTargetCheck = denyUnregisteredTarget | ||
|
|
||
| // denyUnregisteredTarget refuses the update of refName to new when no pack that | ||
| // the registry lists for repo holds the object. It reads the cache repository | ||
| // through a RegisteredPackView, so only the registered packs answer. A registered | ||
| // pack that the cache lacks (a fold replaced the packs after the cache was | ||
| // hydrated) is downloaded first. An object that is no commit is left to the | ||
| // other checks: only "missing" is refused here. | ||
| func denyUnregisteredTarget(ctx context.Context, st ref.RefStore, repo, refName, new string) error { | ||
| repoPath, err := os.Getwd() // git runs hooks in the repository | ||
| if err != nil { | ||
| return fmt.Errorf("update: find the repository: %w", err) | ||
| } | ||
| view, err := NewRegisteredPackView(ctx, st, repoPath, repo, func(ctx context.Context) error { | ||
| bl, err := blob.Open(ctx, mustEnv(envHookBlobURL...)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer bl.Close() | ||
| return HydratePacks(ctx, st, bl, repoPath, repo) | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("update: read the registered packs: %w", err) | ||
| } | ||
| defer view.Close() | ||
| var out strings.Builder | ||
| cmd := exec.CommandContext(ctx, "git", "cat-file", "--batch-check=%(objecttype)") | ||
| cmd.Env = gitlimit.NoReplaceObjects(view.Env(os.Environ())) | ||
| cmd.Stdin = strings.NewReader(new + "\n") | ||
| cmd.Stdout = &out | ||
| if err := cmd.Run(); err != nil { | ||
| return fmt.Errorf("update: read the object type of %s in the registered packs: %w", new, err) | ||
| } | ||
| kinds, err := ParseObjectKinds([]string{new}, out.String()) | ||
| if err != nil { | ||
| return fmt.Errorf("update: %w", err) | ||
| } | ||
| if kinds[new] == "missing" { | ||
| return refusef("update: %s is at %s, which no pack of this repository's registry holds", refName, new) | ||
|
Comment on lines
+2970
to
+2975
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a crafted incoming pack contains the new commit or tag object but omits a referenced tree, parent, or tag target that exists only in this node's cache, receive-pack's connectivity check succeeds and the pack is registered. Since Useful? React with 👍 / 👎. |
||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // objectInMainStore reports whether sha is resolvable WITHOUT the | ||
| // quarantine. git runs pre-receive with GIT_OBJECT_DIRECTORY pointed at the | ||
| // quarantine and the main store as an alternate (githooks(5)); stripping | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For a multi-ref
--atomicpush where an earlier command targets a registered object and a later command targets a cache-only object, the earlier update hook commits its authority CAS before this later per-ref check rejects. Git then aborts the local atomic ref transaction and reports the entire push as failed, while the earlier authority update remains committed. The registered-target check needs a whole-push pre-receive gate for atomic pushes, before any update hook can run.Useful? React with 👍 / 👎.