Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions internal/gitclone/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ package gitclone
import (
"bufio"
"context"
"os"
"os/exec"
"strings"

"github.com/alecthomas/errors"
)

// GitCommand returns a git subprocess configured with repository-scoped
// authentication and any per-URL git config overrides disabled.
// GitCommand returns a Git subprocess with repository hooks disabled.
func (r *Repository) GitCommand(ctx context.Context, args ...string) (*exec.Cmd, error) {
repoURL := r.upstreamURL
var token string
Expand All @@ -30,7 +30,8 @@ func (r *Repository) GitCommand(ctx context.Context, args ...string) (*exec.Cmd,
return nil, errors.Wrap(err, "get insteadOf disable args")
}

var allArgs []string
// Cache operations do not need hooks, and enforcing this at execution prevents restored hook files from running.
allArgs := []string{"-c", "core.hooksPath=" + os.DevNull}
if len(configArgs) > 0 {
allArgs = append(allArgs, configArgs...)
}
Expand Down
194 changes: 165 additions & 29 deletions internal/gitclone/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (

type State int

var errUnsafeMirrorMetadata = errors.New("unsafe mirror metadata")

const (
StateEmpty State = iota // Not cloned yet
StateCloning // Clone in progress
Expand Down Expand Up @@ -180,7 +182,7 @@ func (m *Manager) Config() Config {
return m.config
}

func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repository, error) {
func (m *Manager) GetOrCreate(ctx context.Context, upstreamURL string) (*Repository, error) {
m.clonesMu.RLock()
repo, exists := m.clones[upstreamURL]
m.clonesMu.RUnlock()
Expand Down Expand Up @@ -211,8 +213,21 @@ func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repositor
}

headFile := filepath.Join(clonePath, "HEAD")
if _, err := os.Stat(headFile); err == nil {
repo.state = StateReady
if _, err := os.Lstat(headFile); err == nil {
// Mirrors can survive from an older process or cache restore, so they must cross the same trust boundary as new restores.
if err := resetMirrorMetadata(ctx, clonePath, repo.upstreamURL); err != nil {
if !errors.Is(err, errUnsafeMirrorMetadata) {
return nil, errors.Wrap(err, "sanitize existing mirror")
}
if removeErr := os.RemoveAll(clonePath); removeErr != nil {
return nil, errors.Join(errors.Wrap(err, "sanitize existing mirror"),
errors.Wrap(removeErr, "remove unsafe existing mirror"))
}
} else if err := repo.configureMirror(ctx, clonePath); err != nil {
return nil, errors.Wrap(err, "configure existing mirror")
} else {
repo.state = StateReady
}
}

repo.fetchSem <- struct{}{}
Expand Down Expand Up @@ -267,32 +282,14 @@ func (m *Manager) DiscoverExisting(ctx context.Context) ([]*Repository, error) {
}
upstreamURL := "https://" + host + "/" + repoPath

repo := &Repository{
state: StateReady,
config: m.config,
path: path,
upstreamURL: upstreamURL,
fetchSem: make(chan struct{}, 1),
credentialProvider: m.credentialProvider,
}
repo.fetchSem <- struct{}{}

if err := configureMirror(ctx, path, m.config.PackThreads); err != nil {
return errors.Wrapf(err, "configure mirror for %s", upstreamURL)
repo, err := m.GetOrCreate(ctx, upstreamURL)
if err != nil {
return errors.Wrapf(err, "discover mirror for %s", upstreamURL)
}

if m.config.Maintenance {
if err := registerMaintenance(ctx, path); err != nil {
return errors.Wrapf(err, "register maintenance for %s", upstreamURL)
}
if repo.State() == StateReady {
discovered = append(discovered, repo)
}

m.clonesMu.Lock()
m.clones[upstreamURL] = repo
m.clonesMu.Unlock()

discovered = append(discovered, repo)

return fs.SkipDir
})

Expand All @@ -310,7 +307,24 @@ func RepoPathFromURL(upstreamURL string) (string, error) {
if err != nil {
return "", errors.Wrap(err, "parse upstream URL")
}
return filepath.Join(parsed.Host, strings.TrimSuffix(parsed.Path, ".git")), nil
if parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" ||
parsed.Fragment != "" || strings.ContainsAny(parsed.Host, `/\`) {
return "", errors.Errorf("unsupported upstream URL %q", upstreamURL)
}
repoPath := strings.TrimSuffix(strings.TrimSuffix(strings.TrimPrefix(parsed.Path, "/"), "/"), ".git")
if repoPath == "" {
return "", errors.Errorf("upstream URL %q has no repository path", upstreamURL)
}
for segment := range strings.SplitSeq(repoPath, "/") {
if segment == "" || segment == "." || segment == ".." || strings.Contains(segment, `\`) {
return "", errors.Errorf("unsupported upstream URL path %q", parsed.Path)
}
}
repoPath = filepath.Join(parsed.Host, filepath.FromSlash(repoPath))
if !filepath.IsLocal(repoPath) {
return "", errors.Errorf("upstream URL path %q is not local", parsed.Path)
}
return repoPath, nil
}

func (m *Manager) clonePathForURL(upstreamURL string) (string, error) {
Expand Down Expand Up @@ -400,9 +414,15 @@ func (r *Repository) WithFetchExclusion(ctx context.Context, fn func() error) er
}
}

// ConfigureMirror configures a git directory at repoPath as a mirror with
// the repository's pack threads and optional maintenance settings.
// ConfigureMirror sanitizes and configures a git directory at repoPath as a mirror.
func (r *Repository) ConfigureMirror(ctx context.Context, repoPath string) error {
if err := resetMirrorMetadata(ctx, repoPath, r.upstreamURL); err != nil {
return errors.Wrap(err, "reset mirror metadata")
}
return r.configureMirror(ctx, repoPath)
}

func (r *Repository) configureMirror(ctx context.Context, repoPath string) error {
if err := configureMirror(ctx, repoPath, r.config.PackThreads); err != nil {
return errors.Wrap(err, "configure mirror")
}
Expand All @@ -414,6 +434,122 @@ func (r *Repository) ConfigureMirror(ctx context.Context, repoPath string) error
return nil
}

func resetMirrorMetadata(ctx context.Context, repoPath, upstreamURL string) error {
configPath := filepath.Join(repoPath, "config")
info, err := os.Lstat(configPath)
if err != nil {
return errors.Wrap(err, "stat mirror config")
}
if !info.Mode().IsRegular() {
return errors.Wrap(errUnsafeMirrorMetadata, "mirror config is not a regular file")
}
// Cached metadata can invoke commands; retain only storage features needed to read objects, then rebuild trusted settings.
configValues, err := mirrorConfigValues(ctx, configPath)
if err != nil {
return err
}
objectFormat := configValues["extensions.objectformat"]
if objectFormat != "" && objectFormat != "sha1" && objectFormat != "sha256" {
return errors.Wrapf(errUnsafeMirrorMetadata, "unsupported mirror object format %q", objectFormat)
}
refStorage := configValues["extensions.refstorage"]
if refStorage != "" && refStorage != "files" && refStorage != "reftable" {
return errors.Wrapf(errUnsafeMirrorMetadata, "unsupported mirror ref storage %q", refStorage)
}
// Cleanup must not resolve through a cached symlink and remove files outside the mirror.
if err := filepath.WalkDir(repoPath, func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.Type()&os.ModeSymlink != 0 || (!entry.IsDir() && !entry.Type().IsRegular()) {
return errors.Wrapf(errUnsafeMirrorMetadata, "unsafe mirror entry %s", path)
}
return nil
}); err != nil {
return errors.Wrap(err, "validate mirror entries")
}

repositoryFormat := "0"
if objectFormat != "" || refStorage != "" {
repositoryFormat = "1"
}
escapedRemoteURL, err := gitConfigValue(upstreamURL)
if err != nil {
return err
}
config := "[core]\n\trepositoryformatversion = " + repositoryFormat + "\n\tfilemode = true\n\tbare = true\n" +
"[remote \"origin\"]\n\turl = " + escapedRemoteURL + "\n\tfetch = +refs/*:refs/*\n\tmirror = true\n"
if objectFormat != "" || refStorage != "" {
config += "[extensions]\n"
}
if objectFormat != "" {
config += "\tobjectFormat = " + objectFormat + "\n"
}
if refStorage != "" {
config += "\trefStorage = " + refStorage + "\n"
}
configFile, err := os.CreateTemp(repoPath, ".cachew-config-*")
if err != nil {
return errors.Wrap(err, "create temporary mirror config")
}
configTempPath := configFile.Name()
defer os.Remove(configTempPath) //nolint:errcheck // best-effort cleanup on failure
if _, err := configFile.WriteString(config); err != nil {
_ = configFile.Close()
return errors.Wrap(err, "write temporary mirror config")
}
if err := configFile.Close(); err != nil {
return errors.Wrap(err, "close temporary mirror config")
}

// Git interprets these paths as configuration, hooks, or object-store indirection rather than inert cached data.
for _, path := range []string{
filepath.Join(repoPath, ".git"),
filepath.Join(repoPath, ".lfsconfig"),
filepath.Join(repoPath, "commondir"),
filepath.Join(repoPath, "config.worktree"),
filepath.Join(repoPath, "hooks"),
filepath.Join(repoPath, "index"),
filepath.Join(repoPath, "objects", "info", "alternates"),
filepath.Join(repoPath, "objects", "info", "http-alternates"),
filepath.Join(repoPath, "modules"),
filepath.Join(repoPath, "worktrees"),
} {
if err := os.RemoveAll(path); err != nil {
return errors.Wrapf(err, "remove unsafe mirror metadata %s", path)
}
}
// #nosec G703 - both paths are created within the validated mirror directory
if err := os.Rename(configTempPath, configPath); err != nil {
return errors.Wrap(err, "replace mirror config")
}
return nil
}

func gitConfigValue(value string) (string, error) {
if strings.ContainsAny(value, "\x00\r\n") {
return "", errors.New("mirror origin contains a control character")
}
value = strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `"`, `\"`)
return `"` + value + `"`, nil
}

func mirrorConfigValues(ctx context.Context, configPath string) (map[string]string, error) {
cmd := exec.CommandContext(ctx, "git", "config", "--file", configPath, "--no-includes", "--null", "--list") // #nosec G204
output, err := cmd.CombinedOutput()
if err != nil {
return nil, errors.Wrapf(err, "read mirror config: %s", string(output))
}
values := map[string]string{}
for entry := range strings.SplitSeq(string(output), "\x00") {
key, value, ok := strings.Cut(entry, "\n")
if ok {
values[strings.ToLower(key)] = value
}
}
return values, nil
}

// MarkReady transitions the repository to StateReady, indicating it is
// up-to-date and can serve requests from the local mirror.
func (r *Repository) MarkReady() {
Expand Down
76 changes: 72 additions & 4 deletions internal/gitclone/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,54 @@ func TestManager_GetOrCreate(t *testing.T) {
assert.True(t, repo == repo2, "expected same repository instance")
}

func TestRepoPathFromURLRejectsTraversal(t *testing.T) {
path, err := RepoPathFromURL("https://github.com/org/repo.git")
assert.NoError(t, err)
assert.Equal(t, filepath.Join("github.com", "org", "repo"), path)

for _, upstream := range []string{
"http://github.com/org/repo",
"https://user@github.com/org/repo",
"https://github.com/org/repo?x",
"https://github.com/../../tmp/repo",
"https://github.com/%2e%2e/%2e%2e/tmp/repo",
"https://github.com/org/repo%2f..%2f..%2ftmp/repo",
} {
_, err := RepoPathFromURL(upstream)
assert.Error(t, err)
}
}

func TestConfigureMirrorSanitizesExecutableMetadata(t *testing.T) {
_, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError})
repoPath := createBareRepo(t, t.TempDir())
hookPath := filepath.Join(repoPath, "hooks", "reference-transaction")
proofPath := filepath.Join(t.TempDir(), "proof")
assert.NoError(t, os.WriteFile(hookPath, []byte("#!/bin/sh\ntouch \""+proofPath+"\"\n"), 0o755))
assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "remote.origin.url", "https://github.com/org/repo").Run())
assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "credential.helper", "!touch "+proofPath).Run())
assert.NoError(t, exec.Command("git", "-C", repoPath, "config", "core.alternateRefsCommand", "touch "+proofPath).Run())
assert.NoError(t, os.WriteFile(filepath.Join(repoPath, "objects", "info", "alternates"), []byte(repoPath+"/objects\n"), 0o600))
assert.NoError(t, os.WriteFile(filepath.Join(repoPath, "commondir"), []byte("payload\n"), 0o600))

repo := &Repository{config: testRepoConfig(), upstreamURL: "https://github.com/org/repo"}
assert.NoError(t, repo.ConfigureMirror(ctx, repoPath))

_, err := os.Lstat(hookPath)
assert.True(t, os.IsNotExist(err))
_, err = os.Lstat(filepath.Join(repoPath, "objects", "info", "alternates"))
assert.True(t, os.IsNotExist(err))
_, err = os.Lstat(filepath.Join(repoPath, "commondir"))
assert.True(t, os.IsNotExist(err))
cmd := exec.Command("git", "-C", repoPath, "config", "--get", "credential.helper")
assert.Error(t, cmd.Run())
output, err := exec.Command("git", "-C", repoPath, "config", "--get", "remote.origin.url").Output()
assert.NoError(t, err)
assert.Equal(t, "https://github.com/org/repo", strings.TrimSpace(string(output)))
_, err = os.Lstat(proofPath)
assert.True(t, os.IsNotExist(err))
}

func TestManager_GetOrCreate_ExistingClone(t *testing.T) {
_, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError})
tmpDir := t.TempDir()
Expand All @@ -133,8 +181,7 @@ func TestManager_GetOrCreate_ExistingClone(t *testing.T) {
assert.NoError(t, err)

repoPath := filepath.Join(tmpDir, "github.com", "user", "repo")
assert.NoError(t, os.MkdirAll(repoPath, 0o755))
assert.NoError(t, os.WriteFile(filepath.Join(repoPath, "HEAD"), []byte("ref: refs/heads/main\n"), 0o644))
assert.NoError(t, exec.Command("git", "init", "--bare", repoPath).Run())

upstreamURL := "https://github.com/user/repo"
repo, err := manager.GetOrCreate(context.Background(), upstreamURL)
Expand All @@ -144,6 +191,27 @@ func TestManager_GetOrCreate_ExistingClone(t *testing.T) {
assert.Equal(t, StateReady, repo.State())
}

func TestManager_GetOrCreate_PreservesExistingCloneWhenSanitizationCanceled(t *testing.T) {
_, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError})
tmpDir := t.TempDir()
manager, err := NewManager(ctx, Config{MirrorRoot: tmpDir}, nil)
assert.NoError(t, err)

repoPath := filepath.Join(tmpDir, "github.com", "user", "repo")
assert.NoError(t, exec.Command("git", "init", "--bare", repoPath).Run())
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()

_, err = manager.GetOrCreate(canceledCtx, "https://github.com/user/repo")
assert.Error(t, err)
assert.True(t, errors.Is(err, context.Canceled))
_, err = os.Stat(filepath.Join(repoPath, "HEAD"))
assert.NoError(t, err)
repo, err := manager.GetOrCreate(context.Background(), "https://github.com/user/repo")
assert.NoError(t, err)
assert.Equal(t, StateReady, repo.State())
}

func TestManager_Get(t *testing.T) {
_, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError})
tmpDir := t.TempDir()
Expand Down Expand Up @@ -763,7 +831,7 @@ func TestRepositoryEnsureRefsUpToDateDoesNotCacheInFlightCheck(t *testing.T) {
started := filepath.Join(binDir, "started")
release := filepath.Join(binDir, "release")
script := fmt.Sprintf(`#!/bin/sh
if [ "$1" = "ls-remote" ]; then
if echo " $* " | grep -q ' ls-remote '; then
touch %q
while [ ! -e %q ]; do sleep 0.01; done
fi
Expand Down Expand Up @@ -839,7 +907,7 @@ func TestRepositoryEnsureRefsUpToDateConcurrentStaleResultInvalidatesFreshResult
staleStarted := filepath.Join(binDir, "stale-started")
releaseStale := filepath.Join(binDir, "release-stale")
script := fmt.Sprintf(`#!/bin/sh
if [ "$1" = "ls-remote" ]; then
if echo " $* " | grep -q ' ls-remote '; then
if mkdir %q 2>/dev/null; then
touch %q
while [ ! -e %q ]; do sleep 0.01; done
Expand Down
Loading
Loading