Use shared history module for repository traversal

go.mod

@@ -5,7 +5,9 @@ go 1.26.0
 toolchain go1.26.7
 
 require (
+	github.com/git-pkgs/history v0.1.1
 	github.com/git-pkgs/licenses v0.8.0
+	github.com/git-pkgs/roles v0.0.0-20260917090927-eb44e8aa553d
 	github.com/spf13/cobra v1.10.2
 )
 
@@ -34,10 +36,10 @@ require (
 	github.com/git-pkgs/spdx v0.3.2 // indirect
 	github.com/git-pkgs/vers v0.7.0 // indirect
 	github.com/github/go-spdx/v2 v2.7.0 // indirect
-	github.com/go-git/go-billy/v6 v6.0.0-alpha.2
-	github.com/go-git/go-git/v6 v6.0.0-alpha.4
+	github.com/go-git/go-billy/v6 v6.0.0-alpha.2 // indirect
+	github.com/go-git/go-git/v6 v6.0.0-alpha.5
 	github.com/kevinburke/ssh_config v1.6.0 // indirect
-	github.com/klauspost/compress v1.20.0
+	github.com/klauspost/compress v1.20.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/package-url/packageurl-go v0.1.7 // indirect
 	github.com/pjbgf/sha1cd v0.6.0 // indirect
@@ -47,7 +49,3 @@ require (
 	golang.org/x/sys v0.47.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
-
-replace github.com/go-git/go-git/v6 => github.com/git-pkgs/go-git/v6 v6.0.0-alpha.4.0.20260906204421-52ea7303c392
-
-replace github.com/go-git/go-billy/v6 => github.com/git-pkgs/go-billy/v6 v6.0.0-alpha.2.0.20260906202738-23c9d86c8d17

go.sum

@@ -17,13 +17,12 @@ github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJ
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
-github.com/git-pkgs/go-billy/v6 v6.0.0-alpha.2.0.20260906202738-23c9d86c8d17 h1:PzXJag1pssE3Hhl/tccdaPOtz5j2eZyl1WyUmai71FU=
-github.com/git-pkgs/go-billy/v6 v6.0.0-alpha.2.0.20260906202738-23c9d86c8d17/go.mod h1:QGOWN4CzmtRfpFeCw5paG3RJyKdTwsfGASkxBVkIaAU=
-github.com/git-pkgs/go-git/v6 v6.0.0-alpha.4.0.20260906204421-52ea7303c392 h1:V4/D19lZAHihN/jCHa5lIL6BobOjhTkWFtsOnq/bhYU=
-github.com/git-pkgs/go-git/v6 v6.0.0-alpha.4.0.20260906204421-52ea7303c392/go.mod h1:2eK2w/x2zu4GWIHvrPUnxOTKkQ1H9Hm40hDPLl1az9Q=
+github.com/git-pkgs/history v0.1.1 h1:WZZXMXSaZxU/JboXtiohfPvIv4RNZibPJuNw/BcWSFI=
+github.com/git-pkgs/history v0.1.1/go.mod h1:wZtYwhJsU5LJFLhrw9Lvx4Ue3ESp7+tv4an9KEeVBns=
 github.com/git-pkgs/licenses v0.8.0 h1:nGeWm02M2ei6CXwsIkGwDIYqLBx3ocwwPDDG5TNLSSM=
 github.com/git-pkgs/licenses v0.8.0/go.mod h1:5GW02cjho+lLik/v2XdyVTTq3aNkMNe3nFAvF75mQTg=
 github.com/git-pkgs/magic v0.3.1 h1:UzjFRyEwJITA/JgznjmIM4VwuBszD2K4Q8XHgkgL+DM=
@@ -34,6 +33,8 @@ github.com/git-pkgs/pom v0.1.7 h1:4yKdtw6eyShtjul6bcZdyz7yLQ+jdrYeYkKbskDGi4c=
 github.com/git-pkgs/pom v0.1.7/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk=
 github.com/git-pkgs/purl v0.1.20 h1:a4qzvUy5mBZ2GGjOQNW2h/ocFqjTjOiTDMv2ONtivmM=
 github.com/git-pkgs/purl v0.1.20/go.mod h1:hthV5mp+Q67HpQ9+LnRLLmsReu5ooyQ5EaJsCGrA8yE=
+github.com/git-pkgs/roles v0.0.0-20260917090927-eb44e8aa553d h1:BbX6xoEro5uIMhVY3l17hr/Zt7MYht8NhuEGSTLFK2Q=
+github.com/git-pkgs/roles v0.0.0-20260917090927-eb44e8aa553d/go.mod h1:dRibEXgftewf7z8HKAfj6taRhqRQm/y0bdgQtk1fjg8=
 github.com/git-pkgs/spdx v0.3.2 h1:PV0w09Bt8rIkAof1+zuo3vxNYKgCoFTxQY9E1Q85PL0=
 github.com/git-pkgs/spdx v0.3.2/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU=
 github.com/git-pkgs/vers v0.7.0 h1:7PD2DKFB8jTDIfiyWXbqYW54iVuNkFCBgXHgxOTdr8A=
@@ -44,8 +45,12 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo=
 github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs=
+github.com/go-git/go-billy/v6 v6.0.0-alpha.2 h1:1Sv5WemXL8CxKrAx1gioJ+uHNb2bZJhiQLfwSZ4Et8c=
+github.com/go-git/go-billy/v6 v6.0.0-alpha.2/go.mod h1:r/bsv9i/iDyyEU8/Z6mjC+YraOVwie1ddfUqBCElKXQ=
 github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1 h1:gmqi2jvsreu0s8JMLylYDFq4sbjHwwlhktMw0DUg3mA=
 github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1/go.mod h1:ECf1MqJlBdYpKggBrOXjo/0EnvRZx6D++I86UYjPgAQ=
+github.com/go-git/go-git/v6 v6.0.0-alpha.5 h1:sE+OlkHgYWNMVmN1s9sR7uyFgsWLtxcNWse/vBYKxRE=
+github.com/go-git/go-git/v6 v6.0.0-alpha.5/go.mod h1:3IjhiZnM+uBmUrOGSeqrJpsmi4Vd0H2NZO/uK2a7d0s=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY=
@@ -67,6 +72,7 @@ github.com/package-url/packageurl-go v0.1.7/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH
 github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU=
 github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
@@ -80,11 +86,9 @@ github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
-github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
-go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
 golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
 golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=

gogit_history.go

@@ -1,153 +1,45 @@
 package main
 
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"strings"
-
-	gogit "github.com/go-git/go-git/v6"
-	"github.com/go-git/go-git/v6/plumbing"
-	"github.com/go-git/go-git/v6/plumbing/object"
-	"github.com/go-git/go-git/v6/storage/filesystem"
-)
+import githistory "github.com/git-pkgs/history"
 
 func walkChangesGoGit(path string, merges bool, visit func(change)) error {
-	r, err := openGoGit(path)
+	r, err := openHistory(path)
 	if err != nil {
 		return err
 	}
 	defer func() { _ = r.Close() }()
-	return walkChangesGoGitStream(r, merges, visit)
-}
-
-func historyRoots(r *gogit.Repository) ([]plumbing.Hash, error) {
-	refs, err := r.References()
-	if err != nil {
-		return nil, err
-	}
-	defer refs.Close()
-	var roots []plumbing.Hash
-	head, err := r.Head()
-	if err != nil && !errors.Is(err, plumbing.ErrReferenceNotFound) {
-		return nil, err
-	}
-	if err == nil {
-		h, ok, err := commitAtReference(r, head.Hash())
-		if err != nil {
-			return nil, err
-		}
-		if ok {
-			roots = append(roots, h)
-		}
-	}
-	for {
-		ref, err := refs.Next()
-		if err == io.EOF {
-			return roots, nil
-		}
-		if err != nil {
-			return nil, err
-		}
-		if !strings.HasPrefix(ref.Name().String(), "refs/") {
-			continue
-		}
-		if strings.HasPrefix(ref.Name().String(), "refs/replace/") {
-			return nil, fmt.Errorf("gogit backend does not support replacement refs")
-		}
-		resolved, err := r.Reference(ref.Name(), true)
-		if err != nil {
-			return nil, err
-		}
-		h, ok, err := commitAtReference(r, resolved.Hash())
-		if err != nil {
-			return nil, fmt.Errorf("reference %s: %w", ref.Name(), err)
-		}
-		if ok {
-			roots = append(roots, h)
-		}
-	}
-}
-
-func commitAtReference(r *gogit.Repository, h plumbing.Hash) (plumbing.Hash, bool, error) {
-	seen := make(map[plumbing.Hash]bool)
-	for {
-		if seen[h] {
-			return plumbing.ZeroHash, false, fmt.Errorf("cyclic tag at %s", h)
-		}
-		seen[h] = true
-		o, err := r.Object(plumbing.AnyObject, h)
-		if err != nil {
-			return plumbing.ZeroHash, false, err
-		}
-		switch v := o.(type) {
-		case *object.Commit:
-			return v.Hash, true, nil
-		case *object.Tag:
-			h = v.Target
-		default:
-			return plumbing.ZeroHash, false, nil
-		}
-	}
-}
-
-func commitSubject(message string) string {
-	var lines []string
-	for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
-		if strings.TrimSpace(line) == "" {
-			break
-		}
-		lines = append(lines, strings.TrimSpace(line))
-	}
-	return strings.Join(lines, " ")
+	return r.WalkChanges(githistory.ChangeOptions{
+		Merges:  merges,
+		Workers: *historyWorkers,
+	}, func(c githistory.Change) error {
+		visit(changeFromHistory(c))
+		return nil
+	})
 }
 
 func isShallowGoGit(path string) (bool, error) {
-	r, err := openGoGit(path)
+	r, err := openHistory(path)
 	if err != nil {
 		return false, err
 	}
 	defer func() { _ = r.Close() }()
-	shallow, err := r.Storer.Shallow()
-	return len(shallow) != 0, err
+	return r.Shallow()
 }
 
-func openGoGit(path string) (*gogit.Repository, error) {
-	for _, name := range []string{"GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"} {
-		if os.Getenv(name) != "" {
-			return nil, fmt.Errorf("gogit backend does not support %s", name)
-		}
+func changeFromHistory(c githistory.Change) change {
+	return change{
+		commit: c.Commit, date: c.Date, subject: c.Subject,
+		oldOID: c.OldOID, newOID: c.NewOID, path: c.Path,
+		oldMode: c.OldMode, newMode: c.NewMode,
+		merge: c.Merge,
 	}
-	r, err := gogit.PlainOpen(path)
-	if err != nil {
-		return nil, err
-	}
-	fs := r.Storer.(*filesystem.Storage).Filesystem()
-	for _, name := range []string{"objects/info/alternates", "info/grafts"} {
-		f, err := fs.Open(name)
-		if errors.Is(err, os.ErrNotExist) {
-			continue
-		}
-		if err == nil {
-			var data []byte
-			data, err = io.ReadAll(f)
-			closeErr := f.Close()
-			if err == nil {
-				err = closeErr
-			}
-			if err == nil && len(strings.TrimSpace(string(data))) > 0 {
-				err = fmt.Errorf("gogit backend does not support %s", name)
-			}
-		}
-		if err != nil {
-			_ = r.Close()
-			return nil, err
-		}
-	}
-	if err := configureGoGit(r); err != nil {
-		_ = r.Close()
-		return nil, err
+}
+
+func changeToHistory(c change) githistory.Change {
+	return githistory.Change{
+		Commit: c.commit, Date: c.date, Subject: c.subject,
+		OldOID: c.oldOID, NewOID: c.newOID, Path: c.path,
+		OldMode: c.oldMode, NewMode: c.newMode,
+		Merge: c.merge,
 	}
-	return r, nil
 }

gogit_history_parallel_test.go

@@ -26,16 +26,16 @@ func TestGitFreeParallelHistory(t *testing.T) {
 				date := fmt.Sprintf("2025-01-%02dT12:00:00Z", 13-n)
 				t.Setenv("GIT_AUTHOR_DATE", date)
 				t.Setenv("GIT_COMMITTER_DATE", date)
-				license := []string{"MIT", testApache}[n%2]
+				license := []string{mitExpression, testApache}[n%2]
 				commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: "+license+"\n", fmt.Sprintf("Change %d", n))
 			}
 			git(t, repo, "gc", "--quiet")
 			for _, workers := range []string{"2", "4"} {
 				args := []string{"--history-workers=" + workers, testMmapFlag, "--readers=4"}
 				for _, mode := range []string{"--details", "--monthly"} {
-					commandArgs := append([]string{"log", repo}, args...)
+					commandArgs := append([]string{testLogCommand, repo}, args...)
 					got := cliWithoutGit(t, append(commandArgs, mode)...)
-					want := cli(t, "log", repo, mode)
+					want := cli(t, testLogCommand, repo, mode)
 					if got != want {
 						t.Fatalf("history differs:\n%s\n%s", got, want)
 					}
@@ -59,16 +59,20 @@ func TestGitFreeParallelHistoryMissingTree(t *testing.T) {
 	if err := os.Remove(filepath.Join(repo, ".git", "objects", hash[:2], hash[2:])); err != nil {
 		t.Fatal(err)
 	}
+	cmd = exec.Command("git", "-C", repo, "cat-file", "-e", hash)
+	if out, err := cmd.CombinedOutput(); err == nil {
+		t.Fatalf("tree %s remains readable after removing its loose object:\n%s", hash, out)
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
 	defer cancel()
-	cmd = exec.CommandContext(ctx, os.Args[0], "scan", repo, "--backend=gogit", "--history-workers=4")
+	cmd = exec.CommandContext(ctx, os.Args[0], testScanCommand, repo, "--backend=gogit", "--history-workers=4")
 	cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "PATH="+t.TempDir())
 	out, err = cmd.CombinedOutput()
 	if ctx.Err() != nil {
 		t.Fatal("history workers did not terminate")
 	}
 	if err == nil || !strings.Contains(string(out), "object not found") {
-		t.Fatalf("missing tree accepted: %v\n%s", err, out)
+		t.Fatalf("missing tree %s accepted: %v\n%s", hash, err, out)
 	}
 }
 
@@ -77,7 +81,7 @@ func BenchmarkParallelHistory(b *testing.B) {
 	b.Cleanup(func() { *backend, *goGitMmap, *historyWorkers = oldBackend, oldMmap, oldWorkers })
 	*backend, *goGitMmap = goGitBackend, true
 	for _, root := range benchmarkRepositoryRoots(b) {
-		for _, workers := range []int{1, 2, 4} {
+		for _, workers := range []int{1, 2, 4, 8} {
 			*historyWorkers = workers
 			b.Run(fmt.Sprint(workers), func(b *testing.B) {
 				b.ReportAllocs()