diff --git a/README.md b/README.md index f5a7fef..b96ca30 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,125 @@ # reuse -Go library for parsing REUSE-compliant projects (spec v3.3). Extracts SPDX license and copyright information from file headers, .license sidecars, REUSE.toml annotations, and .reuse/dep5 files. +Go library for reading REUSE licensing metadata from file headers, `.license` +sidecars, `REUSE.toml` annotations, and `.reuse/dep5` files. SPDX expressions +stay as raw strings; use [spdx](https://github.com/git-pkgs/spdx) to validate +them. The package does not check whether a project is REUSE compliant. -SPDX expressions are stored as raw strings. Validation is left to consumers (see [github.com/git-pkgs/spdx](https://github.com/git-pkgs/spdx) if you need it). - -## Installation - -```bash +```sh go get github.com/git-pkgs/reuse ``` -## Usage - -### Parse a project +## Read a project ```go -import "github.com/git-pkgs/reuse" +import ( + "fmt" + "log" -p, err := reuse.OpenProject("/path/to/repo") + "github.com/git-pkgs/reuse" +) -// Get licensing info for a single file. -info, err := p.ReuseInfoOf("src/main.go") -fmt.Println(info.LicenseExpressions) // ["MIT"] -fmt.Println(info.CopyrightNotices) // ["2024 Jane Doe "] - -// Walk all covered files. -all, err := p.AllReuseInfo() -for path, info := range all { - fmt.Printf("%s: %v\n", path, info.LicenseExpressions) +project, err := reuse.OpenProject("/path/to/repo") +if err != nil { + log.Fatal(err) +} +info, err := project.ReuseInfoOf("src/main.go") +if err != nil { + log.Fatal(err) } +fmt.Println(info.LicenseExpressions) +fmt.Println(info.CopyrightNotices) ``` -### Extract SPDX tags from text +`OpenProject` loads annotation files once. `Project.ReuseTOMLs` maps relative +directory names to parsed TOML files, with `"."` for the root. Each lookup reads +the source file or its sidecar; reopen the project after changing annotations. +Paths passed to `ReuseInfoOf` must be relative to the project root. -```go -info := reuse.ExtractReuseInfo(`// SPDX-License-Identifier: MIT OR Apache-2.0 -// SPDX-FileCopyrightText: 2024 Alice`) +A sidecar replaces the source file's header, then participates in annotation +resolution. The outermost matching `override` suppresses closer annotations +and both header and sidecar. Otherwise, the nearest `closest` values fill +missing licence and copyright fields separately. `aggregate` adds values; +ancestor aggregates still apply above an override. Nested TOML patterns are +relative to their own directory, and the last matching table within one file +is used. These rules follow the [REUSE specification](https://reuse.software/spec-3.3/#reuse-toml). -fmt.Println(info.LicenseExpressions) // ["MIT OR Apache-2.0"] -fmt.Println(info.CopyrightNotices) // ["2024 Alice"] -``` +When no TOML files are present, `.reuse/dep5` entries are aggregated with the +header or sidecar. `SourcePath` and `SourceType` identify the primary source; +the combined result does not retain provenance for every individual value. + +`project.AllReuseInfo()` returns a map for covered files, including empty +results when no metadata was found. The walk skips symlinks, empty files, +licence files, sidecars, SPDX documents, and the built-in ignored directories. +It does not apply `.gitignore` or exclude VCS submodules and Meson subprojects. +Use individual lookups when the caller has its own file-selection policy. + +## Read text or annotation files -### Parse REUSE.toml +Text extraction is in the `extract` subpackage: ```go -rt, err := reuse.ParseReuseTOMLFile("REUSE.toml") +import "github.com/git-pkgs/reuse/extract" -info, precedence, ok := rt.ReuseInfoOf("src/main.go") -if ok { - fmt.Println(info.LicenseExpressions) - fmt.Println(precedence) // "closest", "aggregate", or "override" -} +info := extract.ExtractReuseInfo(`// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: 2024 Alice`) +fmt.Println(info.LicenseExpressions) // [MIT OR Apache-2.0] ``` -### Parse .reuse/dep5 +`ExtractReuseInfo` requires complete text and treats its final line as +terminated. Passing a prefix ending at `SPDX-License-Identifier: MIT` can +therefore lose a later `OR Apache-2.0`. Use `peek` for bounded-prefix claims, +and keep those observations separate from resolved project metadata. + +The extractor handles REUSE ignore blocks and collects snippet declarations +alongside file declarations. It does not retain snippet scopes or byte spans. +Empty tags are ignored, and values do not continue onto subsequent lines. +Returned values own their strings, so retaining a short header does not retain +the whole input. `extract.ExtractFromFile` reads the complete file and returns +empty metadata for NUL bytes or invalid UTF-8 in its first 8 KiB, extending that +probe just enough to finish a crossing code point. UTF-16 is not decoded. + +The `toml` subpackage reads annotations without opening a project: ```go -d, err := reuse.ParseDep5File(".reuse/dep5") +import "github.com/git-pkgs/reuse/toml" -info, ok := d.ReuseInfoOf("docs/guide.md") +config, err := toml.ParseReuseTOMLFile("REUSE.toml") +if err != nil { + log.Fatal(err) +} +info, precedence, ok := config.ReuseInfoOf("src/main.go") if ok { - fmt.Println(info.LicenseExpressions) + fmt.Println(info.LicenseExpressions, precedence) } ``` +For DEP5, use `dep5.ParseDep5File` from `github.com/git-pkgs/reuse/dep5`. +Both subpackages also provide string parsers, `toml.ParseReuseTOML` and +`dep5.ParseDep5`. Parsed examples are included in the Go tests. + +File reads and traversal have no size or file-count limits. `AllReuseInfo` +retains all results; applications processing large trees can use individual +lookups to consume results as they go. Concurrent lookups are supported while +the project and its parsed annotations remain unchanged. + ## Development -```bash -git clone --recurse-submodules https://github.com/git-pkgs/reuse -go test -v -race ./... +Initialize the example fixture after cloning, then run the tests and benchmarks: + +```sh +git submodule update --init +go test -race ./... +go test ./extract -run '^$' -bench BenchmarkExtractReuseInfo -benchmem +go test ./extract -run '^$' -bench BenchmarkRetainedHeaders -benchtime 1x +go test . -run '^$' -bench BenchmarkProject -benchmem ``` +The extraction benchmarks cover short and long inputs, ignore blocks, and +snippets. `BenchmarkRetainedHeaders` reports heap retained after collecting +short licence values from separate 128 KiB inputs. `BenchmarkProject` includes +file reads and annotation resolution; its results depend on filesystem caches. + ## License [MIT](LICENSE). diff --git a/dep5/example_test.go b/dep5/example_test.go new file mode 100644 index 0000000..19857cb --- /dev/null +++ b/dep5/example_test.go @@ -0,0 +1,26 @@ +package dep5_test + +import ( + "fmt" + + "github.com/git-pkgs/reuse/dep5" +) + +func ExampleParseDep5() { + config, err := dep5.ParseDep5(`Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ + +Files: docs/* +Copyright: 2024 Alice +License: CC-BY-4.0 +`) + if err != nil { + fmt.Println(err) + return + } + info, ok := config.ReuseInfoOf("docs/guide.md") + if ok { + fmt.Println(info.LicenseExpressions) + } + // Output: + // [CC-BY-4.0] +} diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..8ac4bad --- /dev/null +++ b/example_test.go @@ -0,0 +1,25 @@ +package reuse_test + +import ( + "fmt" + + "github.com/git-pkgs/reuse" +) + +func ExampleProject_ReuseInfoOf() { + project, err := reuse.OpenProject("testdata/fake_repository") + if err != nil { + fmt.Println(err) + return + } + info, err := project.ReuseInfoOf("src/main.go") + if err != nil { + fmt.Println(err) + return + } + fmt.Println(info.LicenseExpressions) + fmt.Println(info.CopyrightNotices) + // Output: + // [MIT] + // [2024 Alice] +} diff --git a/extract/benchmark_test.go b/extract/benchmark_test.go new file mode 100644 index 0000000..2e3b277 --- /dev/null +++ b/extract/benchmark_test.go @@ -0,0 +1,50 @@ +package extract_test + +import ( + "runtime" + "strings" + "testing" + + "github.com/git-pkgs/reuse/extract" +) + +func BenchmarkExtractReuseInfo(b *testing.B) { + for _, tc := range []struct { + name string + text string + }{ + {"plain-1KiB", strings.Repeat("x", 1024)}, + {"plain-64KiB", strings.Repeat("x", 64<<10)}, + {"header-1KiB", "// SPDX-License-Identifier: MIT\n" + strings.Repeat("x", 1024)}, + {"header-64KiB", "// SPDX-License-Identifier: MIT\n" + strings.Repeat("x", 64<<10)}, + {"ignored", "REUSE-IgnoreStart\nSPDX-License-Identifier: GPL-3.0-only\nREUSE-IgnoreEnd\nSPDX-License-Identifier: MIT\n"}, + {"snippet", "SPDX-License-Identifier: MIT\nSPDX-SnippetBegin\nSPDX-License-Identifier: Apache-2.0\nSPDX-SnippetEnd\n"}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(tc.text))) + for b.Loop() { + _ = extract.ExtractReuseInfo(tc.text) + } + }) + } +} + +func BenchmarkRetainedHeaders(b *testing.B) { + const count = 64 + text := "// SPDX-License-Identifier: MIT\n" + strings.Repeat("x", 128<<10) + for b.Loop() { + values := make([]string, 0, count) + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + for range count { + info := extract.ExtractReuseInfo(strings.Clone(text)) + values = append(values, info.LicenseExpressions...) + } + runtime.GC() + runtime.ReadMemStats(&after) + b.ReportMetric(float64(max(0, int64(after.HeapAlloc)-int64(before.HeapAlloc)))/count, "retained-B/file") + runtime.KeepAlive(values) + } +} diff --git a/extract/example_test.go b/extract/example_test.go new file mode 100644 index 0000000..c758dc2 --- /dev/null +++ b/extract/example_test.go @@ -0,0 +1,17 @@ +package extract_test + +import ( + "fmt" + + "github.com/git-pkgs/reuse/extract" +) + +func ExampleExtractReuseInfo() { + info := extract.ExtractReuseInfo(`// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: 2024 Alice`) + fmt.Println(info.LicenseExpressions) + fmt.Println(info.CopyrightNotices) + // Output: + // [MIT OR Apache-2.0] + // [2024 Alice] +} diff --git a/extract/extract.go b/extract/extract.go index 3abccc2..ae5fbcd 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -12,18 +12,18 @@ import ( var ( // Trailing comment-end markers stripped from matched values. - endPattern = `[\s]*(?:\*/|-->|"|'>|\]\s*::|"\))?[\s]*$` + endPattern = `[ \t\f]*(?:\*/|-->|"|'>|\][ \t\f]*::|"\))?[ \t\f]*$` licensePattern = regexp.MustCompile( - `(?m)^.*?SPDX-License-Identifier:\s*(?P.*?)` + endPattern, + `(?m)SPDX-License-Identifier:[ \t\f]*(?P.*?)` + endPattern, ) copyrightPattern = regexp.MustCompile( - `(?m)^.*?SPDX-(?:File|Snippet)CopyrightText:\s*(?P.*?)` + endPattern, + `(?m)SPDX-(?:File|Snippet)CopyrightText:[ \t\f]*(?P.*?)` + endPattern, ) contributorPattern = regexp.MustCompile( - `(?m)^.*?SPDX-FileContributor:\s*(?P.*?)` + endPattern, + `(?m)SPDX-FileContributor:[ \t\f]*(?P.*?)` + endPattern, ) snippetBeginPattern = regexp.MustCompile(`(?m)^.*?SPDX-SnippetBegin`) @@ -32,7 +32,8 @@ var ( // ExtractReuseInfo extracts SPDX license, copyright, and contributor // information from the given text. It handles REUSE-IgnoreStart/End blocks -// and SPDX-SnippetBegin/End regions. +// and SPDX-SnippetBegin/End regions. The text must be complete; the final line +// is treated as terminated. Returned values do not retain the input string. func ExtractReuseInfo(text string) core.ReuseInfo { // Normalize line endings. text = strings.ReplaceAll(text, "\r\n", "\n") @@ -40,6 +41,9 @@ func ExtractReuseInfo(text string) core.ReuseInfo { // Strip ignore blocks. text = FilterIgnoreBlocks(text) + if !strings.Contains(text, "SPDX-") { + return core.ReuseInfo{} + } // Strip snippet regions but keep the snippet tags themselves for extraction. // The spec says snippet info applies only to the snippet, but for file-level @@ -59,19 +63,19 @@ func extractFromText(text string, info *core.ReuseInfo) { for _, m := range licensePattern.FindAllStringSubmatch(text, -1) { val := strings.TrimSpace(m[1]) if val != "" { - info.LicenseExpressions = append(info.LicenseExpressions, val) + info.LicenseExpressions = append(info.LicenseExpressions, strings.Clone(val)) } } for _, m := range copyrightPattern.FindAllStringSubmatch(text, -1) { val := strings.TrimSpace(m[1]) if val != "" { - info.CopyrightNotices = append(info.CopyrightNotices, val) + info.CopyrightNotices = append(info.CopyrightNotices, strings.Clone(val)) } } for _, m := range contributorPattern.FindAllStringSubmatch(text, -1) { val := strings.TrimSpace(m[1]) if val != "" { - info.Contributors = append(info.Contributors, val) + info.Contributors = append(info.Contributors, strings.Clone(val)) } } } @@ -80,6 +84,9 @@ func extractFromText(text string, info *core.ReuseInfo) { // regions and the snippet regions themselves. If no snippets are found, the // entire text is returned as the file portion. func splitSnippets(text string) (string, []string) { + if !strings.Contains(text, "SPDX-SnippetBegin") { + return text, nil + } beginLocs := snippetBeginPattern.FindAllStringIndex(text, -1) if len(beginLocs) == 0 { return text, nil @@ -180,7 +187,13 @@ const binaryCheckLimit = 8192 func isBinary(data []byte) bool { check := data if len(check) > binaryCheckLimit { - check = check[:binaryCheckLimit] + // Include a code point that straddles the probe boundary. + start := binaryCheckLimit - 1 + for start > binaryCheckLimit-utf8.UTFMax && !utf8.RuneStart(data[start]) { + start-- + } + _, size := utf8.DecodeRune(data[start:]) + check = check[:max(binaryCheckLimit, start+size)] } if slices.Contains(check, 0) { diff --git a/extract/line_test.go b/extract/line_test.go new file mode 100644 index 0000000..ea06e3a --- /dev/null +++ b/extract/line_test.go @@ -0,0 +1,31 @@ +package extract_test + +import ( + "slices" + "strings" + "testing" + + "github.com/git-pkgs/reuse/extract" +) + +func TestExtractReuseInfoEmptyTags(t *testing.T) { + for _, ending := range []struct{ name, text string }{ + {"LF", "\n"}, {"CRLF", "\r\n"}, {"CR", "\r"}, + } { + for _, tag := range []string{ + "SPDX-License-Identifier", "SPDX-FileCopyrightText", + "SPDX-SnippetCopyrightText", "SPDX-FileContributor", + } { + t.Run(ending.name+"/"+tag, func(t *testing.T) { + text := strings.Join([]string{ + tag + ": \t", "", tag + ": valid", tag + ":", "ordinary text", + }, ending.text) + info := extract.ExtractReuseInfo(text) + values := slices.Concat(info.LicenseExpressions, info.CopyrightNotices, info.Contributors) + if !slices.Equal(values, []string{"valid"}) { + t.Fatalf("values = %q, want [valid]", values) + } + }) + } + } +} diff --git a/internal/core/core.go b/internal/core/core.go index 2990eea..3cf3da5 100644 --- a/internal/core/core.go +++ b/internal/core/core.go @@ -56,8 +56,8 @@ func (p PrecedenceType) String() string { } } -// ReuseInfo holds the licensing and copyright information extracted from a -// single source (file header, sidecar, REUSE.toml annotation, or dep5 paragraph). +// ReuseInfo holds raw licensing metadata. For combined results, SourcePath and +// SourceType identify the primary source rather than every contributing source. type ReuseInfo struct { LicenseExpressions []string CopyrightNotices []string diff --git a/project.go b/project.go index 378c4b6..d1ad303 100644 --- a/project.go +++ b/project.go @@ -1,9 +1,13 @@ package reuse import ( + "errors" + "fmt" "os" "path/filepath" + "slices" "strings" + "syscall" "github.com/git-pkgs/reuse/dep5" "github.com/git-pkgs/reuse/extract" @@ -13,28 +17,38 @@ import ( // Project represents a REUSE-compliant project directory. type Project struct { Root string - ReuseTOML *rtoml.ReuseTOML + ReuseTOMLs map[string]*rtoml.ReuseTOML // keyed by relative directory; "." is the root Dep5 *dep5.Dep5 LicenseFiles []string // relative paths in LICENSES/ } // OpenProject discovers and parses REUSE metadata in the given project root. -// It looks for REUSE.toml, .reuse/dep5, and LICENSES/ directory. +// It loads REUSE.toml files throughout the covered directories, or .reuse/dep5 +// when no TOML files are present, and lists LICENSES/. func OpenProject(root string) (*Project, error) { - p := &Project{Root: root} - - // Try REUSE.toml. - tomlPath := filepath.Join(root, "REUSE.toml") - if _, err := os.Stat(tomlPath); err == nil { - rt, err := rtoml.ParseReuseTOMLFile(tomlPath) - if err != nil { - return nil, err - } - p.ReuseTOML = rt + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + return nil, err + } + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("project root is not a directory: %s", root) } + tomls, err := readReuseTOMLs(root) + if err != nil { + return nil, err + } + p := &Project{Root: root, ReuseTOMLs: tomls} // Try .reuse/dep5 (only if no REUSE.toml, they are mutually exclusive). - if p.ReuseTOML == nil { + if len(p.ReuseTOMLs) == 0 { dep5Path := filepath.Join(root, ".reuse", "dep5") if _, err := os.Stat(dep5Path); err == nil { d, err := dep5.ParseDep5File(dep5Path) @@ -64,65 +78,61 @@ func OpenProject(root string) (*Project, error) { return p, nil } -// ReuseInfoOf resolves all REUSE information sources for the given path -// (relative to project root) and returns the combined result. The precedence -// order follows the spec: -// -// 1. .license sidecar file (if present, used exclusively) -// 2. REUSE.toml override annotations -// 3. File header extraction -// 4. REUSE.toml closest/aggregate annotations -// 5. dep5 -func (p *Project) ReuseInfoOf(path string) (ReuseInfo, error) { - // 1. Check for .license sidecar. - sidecarPath := filepath.Join(p.Root, path+".license") - if fi, err := os.Stat(sidecarPath); err == nil && fi.Mode().IsRegular() { - info, err := extract.ExtractFromFile(sidecarPath) +func readReuseTOMLs(root string) (map[string]*rtoml.ReuseTOML, error) { + files := make(map[string]*rtoml.ReuseTOML) + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { if err != nil { - return ReuseInfo{}, err + return err } - info.SourceType = DotLicense - info.SourcePath = path + ".license" - return info, nil - } - - // 2. Check REUSE.toml for override. - if p.ReuseTOML != nil { - tomlInfo, prec, ok := p.ReuseTOML.ReuseInfoOf(path) - if ok && prec == Override { - return tomlInfo, nil + if entry.IsDir() { + if path != root && IsIgnoredDir(entry.Name()) { + return filepath.SkipDir + } + return nil } - } + if entry.Name() != "REUSE.toml" || !entry.Type().IsRegular() { + return nil + } + parsed, err := rtoml.ParseReuseTOMLFile(path) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + parsed.Source = filepath.ToSlash(relative) + files[filepath.ToSlash(filepath.Dir(relative))] = parsed + return nil + }) + return files, err +} - // 3. Extract from file header. - filePath := filepath.Join(p.Root, path) - fileInfo, err := extract.ExtractFromFile(filePath) +// ReuseInfoOf resolves all REUSE information sources for the given path +// (relative to project root) and returns the combined result. A sidecar replaces +// the file header. The outermost TOML override suppresses closer sources; +// otherwise closest annotations fill missing fields and aggregate adds values. +func (p *Project) ReuseInfoOf(path string) (ReuseInfo, error) { + if !filepath.IsLocal(path) { + return ReuseInfo{}, fmt.Errorf("path must be relative to the project: %s", path) + } + path = filepath.Clean(path) + closest, aggregate, override := p.annotationsFor(path) + if override != nil { + return mergeReuseInfo(*override, aggregate), nil + } + fileInfo, err := p.fileInfo(path) if err != nil { - // If the file can't be read, still try other sources. - if !os.IsNotExist(err) { - return ReuseInfo{}, err - } - fileInfo = ReuseInfo{} + return ReuseInfo{}, err } - fileInfo.SourcePath = path - fileInfo.SourceType = FileHeader - - // 4. Apply REUSE.toml closest/aggregate annotations. - if p.ReuseTOML != nil { - tomlInfo, prec, ok := p.ReuseTOML.ReuseInfoOf(path) - if ok { - switch prec { - case Aggregate: - fileInfo = mergeReuseInfo(fileInfo, tomlInfo) - case Closest: - fileInfo = applyClosest(fileInfo, tomlInfo) - } - } + if !closest.IsEmpty() { + fileInfo = applyClosest(fileInfo, closest) } + fileInfo = mergeReuseInfo(fileInfo, aggregate) - // 5. Check dep5 (aggregate precedence). + // DEP5 uses aggregate precedence. if p.Dep5 != nil { - dep5Info, ok := p.Dep5.ReuseInfoOf(path) + dep5Info, ok := p.Dep5.ReuseInfoOf(filepath.ToSlash(path)) if ok { dep5Info.SourcePath = filepath.Join(".reuse", "dep5") if fileInfo.IsEmpty() { @@ -135,6 +145,58 @@ func (p *Project) ReuseInfoOf(path string) (ReuseInfo, error) { return fileInfo, nil } +func (p *Project) annotationsFor(path string) (closest, aggregate ReuseInfo, override *ReuseInfo) { + var directories []string + for dir := filepath.Dir(path); ; dir = filepath.Dir(dir) { + directories = append(directories, dir) + if dir == "." { + break + } + } + slices.Reverse(directories) + for _, dir := range directories { + config := p.ReuseTOMLs[filepath.ToSlash(dir)] + if config == nil { + continue + } + relative := filepath.ToSlash(path) + if dir != "." { + relative = strings.TrimPrefix(relative, filepath.ToSlash(dir)+"/") + } + info, precedence, ok := config.ReuseInfoOf(relative) + if !ok { + continue + } + switch precedence { + case Closest: + closest = applyClosest(info, closest) + case Aggregate: + aggregate = mergeReuseInfo(aggregate, info) + case Override: + return ReuseInfo{}, aggregate, &info + } + } + return closest, aggregate, nil +} + +func (p *Project) fileInfo(path string) (ReuseInfo, error) { + source := path + sourceType := FileHeader + if info, err := os.Stat(filepath.Join(p.Root, path+".license")); err == nil && info.Mode().IsRegular() { + source += ".license" + sourceType = DotLicense + } else if err != nil && !os.IsNotExist(err) && !errors.Is(err, syscall.ENAMETOOLONG) { + return ReuseInfo{}, err + } + info, err := extract.ExtractFromFile(filepath.Join(p.Root, source)) + if err != nil && (sourceType == DotLicense || !os.IsNotExist(err)) { + return ReuseInfo{}, err + } + info.SourcePath = filepath.ToSlash(source) + info.SourceType = sourceType + return info, nil +} + // AllReuseInfo walks all covered files in the project and returns licensing // information for each one, keyed by relative path. func (p *Project) AllReuseInfo() (map[string]ReuseInfo, error) { @@ -158,6 +220,9 @@ func (p *Project) AllReuseInfo() (map[string]ReuseInfo, error) { // mergeReuseInfo combines two ReuseInfo values, appending licenses, copyrights, // and contributors from other into base. func mergeReuseInfo(base, other ReuseInfo) ReuseInfo { + if base.IsEmpty() && len(base.Contributors) == 0 && !other.IsEmpty() { + base.SourcePath, base.SourceType = other.SourcePath, other.SourceType + } base.LicenseExpressions = appendUnique(base.LicenseExpressions, other.LicenseExpressions...) base.CopyrightNotices = appendUnique(base.CopyrightNotices, other.CopyrightNotices...) base.Contributors = appendUnique(base.Contributors, other.Contributors...) @@ -168,7 +233,7 @@ func mergeReuseInfo(base, other ReuseInfo) ReuseInfo { // annotation, but only if the file is missing that particular piece. func applyClosest(fileInfo, tomlInfo ReuseInfo) ReuseInfo { if !fileInfo.HasLicense() && !fileInfo.HasCopyright() { - // File has no REUSE info at all; use the toml info entirely. + tomlInfo.Contributors = appendUnique(fileInfo.Contributors, tomlInfo.Contributors...) return tomlInfo } if !fileInfo.HasLicense() { @@ -181,6 +246,9 @@ func applyClosest(fileInfo, tomlInfo ReuseInfo) ReuseInfo { } func appendUnique(base []string, items ...string) []string { + if len(items) == 0 { + return base + } seen := make(map[string]bool, len(base)) for _, s := range base { seen[s] = true diff --git a/project_benchmark_test.go b/project_benchmark_test.go new file mode 100644 index 0000000..6189601 --- /dev/null +++ b/project_benchmark_test.go @@ -0,0 +1,47 @@ +package reuse_test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/git-pkgs/reuse" +) + +func BenchmarkProject(b *testing.B) { + root := b.TempDir() + text := "// SPDX-License-Identifier: MIT\n" + strings.Repeat("// source line\n", 4096) + for i := range 16 { + if err := os.WriteFile(filepath.Join(root, fmt.Sprintf("file%d.go", i)), []byte(text), 0600); err != nil { + b.Fatal(err) + } + } + config := "version = 1\n[[annotations]]\npath = \"*.go\"\nSPDX-FileCopyrightText = \"2024 Example\"\n" + if err := os.WriteFile(filepath.Join(root, "REUSE.toml"), []byte(config), 0600); err != nil { + b.Fatal(err) + } + project, err := reuse.OpenProject(root) + if err != nil { + b.Fatal(err) + } + b.Run("ReuseInfoOf", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + info, err := project.ReuseInfoOf("file0.go") + if err != nil || !info.HasLicense() { + b.Fatalf("info=%+v err=%v", info, err) + } + } + }) + b.Run("AllReuseInfo", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + info, err := project.AllReuseInfo() + if err != nil || len(info) != 16 { + b.Fatalf("files=%d err=%v", len(info), err) + } + } + }) +} diff --git a/project_empty_tags_test.go b/project_empty_tags_test.go new file mode 100644 index 0000000..913c9e6 --- /dev/null +++ b/project_empty_tags_test.go @@ -0,0 +1,56 @@ +package reuse_test + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/git-pkgs/reuse" +) + +func TestProjectEmptyTagsAllowAnnotationFallback(t *testing.T) { + for _, sidecar := range []bool{false, true} { + name := "header" + if sidecar { + name = "sidecar" + } + t.Run(name, func(t *testing.T) { + root := t.TempDir() + files := map[string]string{ + "REUSE.toml": "version = 1\n[[annotations]]\npath = \"*.go\"\n" + + "SPDX-License-Identifier = \"MIT\"\nSPDX-FileCopyrightText = \"2024 Author\"\n", + "main.go": "// SPDX-License-Identifier: \t\n// SPDX-FileCopyrightText:\n" + + "// SPDX-FileContributor:\npackage main\n", + } + if sidecar { + files["main.go.license"] = "SPDX-License-Identifier:\n\nSPDX-FileCopyrightText:\n" + + "SPDX-FileContributor:\nSPDX-FileContributor: Contributor\n" + } + for path, text := range files { + if err := os.WriteFile(filepath.Join(root, path), []byte(text), 0600); err != nil { + t.Fatal(err) + } + } + project, err := reuse.OpenProject(root) + if err != nil { + t.Fatal(err) + } + info, err := project.ReuseInfoOf("main.go") + if err != nil { + t.Fatal(err) + } + if !slices.Equal(info.LicenseExpressions, []string{"MIT"}) || + !slices.Equal(info.CopyrightNotices, []string{"2024 Author"}) { + t.Fatalf("annotation fallback: %+v", info) + } + var contributors []string + if sidecar { + contributors = []string{"Contributor"} + } + if !slices.Equal(info.Contributors, contributors) { + t.Fatalf("contributors = %q, want %q", info.Contributors, contributors) + } + }) + } +} diff --git a/project_encoding_test.go b/project_encoding_test.go new file mode 100644 index 0000000..b982091 --- /dev/null +++ b/project_encoding_test.go @@ -0,0 +1,56 @@ +package reuse_test + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/git-pkgs/reuse" +) + +func TestProjectUTF8ProbeBoundary(t *testing.T) { + const boundary = 8192 + const header = "// SPDX-License-Identifier: MIT\n" + for _, character := range []string{"é", "€", "\U00010400"} { + for split := 1; split < len(character); split++ { + text := header + strings.Repeat(" ", boundary-split-len(header)) + character + "\n" + t.Run(character+strings.Repeat("_", split), func(t *testing.T) { + checkProjectLicense(t, text, []string{"MIT"}) + }) + } + } + for _, tc := range []struct { + name string + text string + }{ + {"invalid-before-boundary", header + "\xff" + strings.Repeat(" ", boundary)}, + {"invalid-at-boundary", header + strings.Repeat(" ", boundary-1-len(header)) + "\xc3x"}, + {"incomplete-at-EOF", header + strings.Repeat(" ", boundary-1-len(header)) + "\xe2\x82"}, + {"NUL", header + "\x00" + strings.Repeat(" ", boundary)}, + } { + t.Run(tc.name, func(t *testing.T) { + checkProjectLicense(t, tc.text, nil) + }) + } +} + +func checkProjectLicense(t *testing.T, text string, want []string) { + t.Helper() + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "source.go"), []byte(text), 0600); err != nil { + t.Fatal(err) + } + project, err := reuse.OpenProject(root) + if err != nil { + t.Fatal(err) + } + info, err := project.ReuseInfoOf("source.go") + if err != nil { + t.Fatal(err) + } + if !slices.Equal(info.LicenseExpressions, want) { + t.Fatalf("licenses = %q, want %q", info.LicenseExpressions, want) + } +} diff --git a/project_resolution_test.go b/project_resolution_test.go new file mode 100644 index 0000000..f437389 --- /dev/null +++ b/project_resolution_test.go @@ -0,0 +1,236 @@ +package reuse + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestProjectAllReuseInfoLongFilename(t *testing.T) { + root := t.TempDir() + const filenameLength = 250 + const extension = ".go" + name := strings.Repeat("a", filenameLength-len(extension)) + extension + mkfile(t, root, name, "// SPDX-License-Identifier: MIT\npackage main\n") + mkfile(t, root, "main.go", "// SPDX-License-Identifier: Apache-2.0\npackage main\n") + project, err := OpenProject(root) + if err != nil { + t.Fatal(err) + } + all, err := project.AllReuseInfo() + if err != nil { + t.Fatal(err) + } + if len(all) != 2 { + t.Fatalf("covered files = %d, want 2", len(all)) + } + assertSlice(t, all[name].LicenseExpressions, []string{"MIT"}) + assertSlice(t, all["main.go"].LicenseExpressions, []string{"Apache-2.0"}) +} + +func TestProjectSidecarPrecedence(t *testing.T) { + for _, tc := range []struct { + precedence string + want []string + }{ + {"closest", []string{"MIT"}}, + {"aggregate", []string{"Apache-2.0", "MIT"}}, + {"override", []string{"Apache-2.0"}}, + } { + t.Run(tc.precedence, func(t *testing.T) { + root := t.TempDir() + mkfile(t, root, "file.go", "// SPDX-License-Identifier: GPL-3.0-only\n") + mkfile(t, root, "file.go.license", "SPDX-License-Identifier: MIT\n") + mkfile(t, root, "REUSE.toml", annotation("*.go", tc.precedence, "Apache-2.0", "2024 Annotation Author")) + project, err := OpenProject(root) + if err != nil { + t.Fatal(err) + } + info, err := project.ReuseInfoOf("file.go") + if err != nil { + t.Fatal(err) + } + slices.Sort(info.LicenseExpressions) + assertSlice(t, info.LicenseExpressions, tc.want) + assertSlice(t, info.CopyrightNotices, []string{"2024 Annotation Author"}) + }) + } +} + +func TestProjectNestedAnnotations(t *testing.T) { + for _, tc := range []struct { + name, outer, inner, header string + want []string + }{ + {"closest", "closest", "closest", "", []string{"Apache-2.0"}}, + {"header", "closest", "closest", "GPL-3.0-only", []string{"GPL-3.0-only"}}, + {"outer aggregate", "aggregate", "closest", "", []string{"Apache-2.0", "MIT"}}, + {"aggregate with header", "aggregate", "closest", "GPL-3.0-only", []string{"GPL-3.0-only", "MIT"}}, + {"inner aggregate", "closest", "aggregate", "", []string{"Apache-2.0", "MIT"}}, + {"inner override", "closest", "override", "GPL-3.0-only", []string{"Apache-2.0"}}, + {"aggregate and override", "aggregate", "override", "GPL-3.0-only", []string{"Apache-2.0", "MIT"}}, + {"outer override", "override", "closest", "GPL-3.0-only", []string{"MIT"}}, + {"two overrides", "override", "override", "GPL-3.0-only", []string{"MIT"}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + mkfile(t, root, "REUSE.toml", annotation("src/**", tc.outer, "MIT", "")) + mkfile(t, root, "src/REUSE.toml", annotation("*.go", tc.inner, "Apache-2.0", "")) + mkfile(t, root, "src/main.go", "// SPDX-License-Identifier: "+tc.header+"\n") + project, err := OpenProject(root) + if err != nil { + t.Fatal(err) + } + info, err := project.ReuseInfoOf(filepath.Join("src", "main.go")) + if err != nil { + t.Fatal(err) + } + slices.Sort(info.LicenseExpressions) + assertSlice(t, info.LicenseExpressions, tc.want) + }) + } +} + +func TestProjectNestedOnlyAndRelativePaths(t *testing.T) { + root := t.TempDir() + mkfile(t, root, "src/REUSE.toml", annotation("*.go", "closest", "MIT", "2024 Nested Author")) + for _, path := range []string{"src/main.go", "src/sub/main.go", "other/main.go"} { + mkfile(t, root, path, "package main\n") + } + project, err := OpenProject(root) + if err != nil { + t.Fatal(err) + } + all, err := project.AllReuseInfo() + if err != nil { + t.Fatal(err) + } + if len(all) != 3 { + t.Fatalf("covered files = %d, want 3", len(all)) + } + for path, info := range all { + if filepath.ToSlash(path) == "src/main.go" { + assertSlice(t, info.LicenseExpressions, []string{"MIT"}) + if info.SourcePath != "src/REUSE.toml" { + t.Fatalf("source = %q", info.SourcePath) + } + } else if !info.IsEmpty() { + t.Fatalf("annotation escaped its scope: %s: %+v", path, info) + } + } +} + +func TestProjectNestedClosestFillsFieldsSeparately(t *testing.T) { + root := t.TempDir() + mkfile(t, root, "REUSE.toml", annotation("**", "closest", "MIT", "2024 Outer Author")) + mkfile(t, root, "src/REUSE.toml", annotation("**", "closest", "", "2024 Inner Author")) + mkfile(t, root, "src/sub/main.go", "package main\n") + project, err := OpenProject(root) + if err != nil { + t.Fatal(err) + } + info, err := project.ReuseInfoOf(filepath.Join("src", "sub", "main.go")) + if err != nil { + t.Fatal(err) + } + assertSlice(t, info.LicenseExpressions, []string{"MIT"}) + assertSlice(t, info.CopyrightNotices, []string{"2024 Inner Author"}) +} + +func TestProjectSidecarDep5Aggregate(t *testing.T) { + root := setupFakeProject(t, "dep5") + mkfile(t, root, "src/main.go.license", "SPDX-License-Identifier: Apache-2.0\n") + project, err := OpenProject(root) + if err != nil { + t.Fatal(err) + } + info, err := project.ReuseInfoOf(filepath.Join("src", "main.go")) + if err != nil { + t.Fatal(err) + } + assertSlice(t, info.LicenseExpressions, []string{"Apache-2.0", "MIT"}) +} + +func TestProjectResultStorage(t *testing.T) { + root := t.TempDir() + mkfile(t, root, "REUSE.toml", annotation("*.go", "closest", "MIT", "2024 Author")) + mkfile(t, root, "main.go", "// SPDX-FileContributor: Contributor\n") + project, err := OpenProject(root) + if err != nil { + t.Fatal(err) + } + for range 8 { + t.Run("worker", func(t *testing.T) { + t.Parallel() + for range 10 { + info, err := project.ReuseInfoOf("main.go") + if err != nil { + t.Fatal(err) + } + assertSlice(t, info.LicenseExpressions, []string{"MIT"}) + assertSlice(t, info.CopyrightNotices, []string{"2024 Author"}) + assertSlice(t, info.Contributors, []string{"Contributor"}) + info.LicenseExpressions[0] = "changed" + info.CopyrightNotices[0] = "changed" + } + }) + } +} + +func TestProjectRejectsNonlocalPaths(t *testing.T) { + project, err := OpenProject(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, path := range []string{"", "../file.go", filepath.Join(project.Root, "file.go")} { + if _, err := project.ReuseInfoOf(path); err == nil { + t.Errorf("accepted nonlocal path %q", path) + } + } +} + +func TestOpenProjectNestedMetadata(t *testing.T) { + root := t.TempDir() + mkfile(t, root, ".git/REUSE.toml", "invalid TOML") + mkfile(t, root, "LICENSES/REUSE.toml", "invalid TOML") + if _, err := OpenProject(root); err != nil { + t.Fatal(err) + } + mkfile(t, root, "src/REUSE.toml", "invalid TOML") + if _, err := OpenProject(root); err == nil { + t.Fatal("accepted malformed nested annotations") + } +} + +func TestOpenProjectSymlinkRoot(t *testing.T) { + root := t.TempDir() + mkfile(t, root, "REUSE.toml", annotation("src/**", "closest", "MIT", "")) + mkfile(t, root, "src/main.go", "package main\n") + link := filepath.Join(t.TempDir(), "project") + if err := os.Symlink(root, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + project, err := OpenProject(link) + if err != nil { + t.Fatal(err) + } + info, err := project.ReuseInfoOf(filepath.Join("src", "main.go")) + if err != nil { + t.Fatal(err) + } + assertSlice(t, info.LicenseExpressions, []string{"MIT"}) +} + +func annotation(path, precedence, license, copyright string) string { + text := fmt.Sprintf("version = 1\n[[annotations]]\npath = %q\nprecedence = %q\n", path, precedence) + if license != "" { + text += fmt.Sprintf("SPDX-License-Identifier = %q\n", license) + } + if copyright != "" { + text += fmt.Sprintf("SPDX-FileCopyrightText = %q\n", copyright) + } + return text +} diff --git a/project_test.go b/project_test.go index ce41ac7..ccda11c 100644 --- a/project_test.go +++ b/project_test.go @@ -14,7 +14,7 @@ func TestOpenProject_WithReuseTOML(t *testing.T) { t.Fatal(err) } - if p.ReuseTOML == nil { + if p.ReuseTOMLs["."] == nil { t.Fatal("expected ReuseTOML to be set") } if p.Dep5 != nil { @@ -33,7 +33,7 @@ func TestOpenProject_WithDep5(t *testing.T) { t.Fatal(err) } - if p.ReuseTOML != nil { + if len(p.ReuseTOMLs) != 0 { t.Error("expected ReuseTOML to be nil") } if p.Dep5 == nil { @@ -49,7 +49,7 @@ func TestOpenProject_HeaderOnly(t *testing.T) { t.Fatal(err) } - if p.ReuseTOML != nil { + if len(p.ReuseTOMLs) != 0 { t.Error("expected ReuseTOML to be nil") } if p.Dep5 != nil { @@ -250,7 +250,7 @@ func Bare() {}`) assertSlice(t, info.CopyrightNotices, []string{"2024 Dep5 Author"}) } -func TestProject_ReuseInfoOf_SidecarOverridesAll(t *testing.T) { +func TestProject_ReuseInfoOf_TOMLOverridesSidecar(t *testing.T) { root := setupFakeProject(t, "toml") mkfile(t, root, "src/special.go", `// SPDX-License-Identifier: MIT @@ -278,9 +278,8 @@ SPDX-License-Identifier = "Apache-2.0" t.Fatal(err) } - // Sidecar takes absolute precedence over everything. - assertSlice(t, info.LicenseExpressions, []string{"GPL-3.0-only"}) - assertSlice(t, info.CopyrightNotices, []string{"2024 Sidecar Author"}) + assertSlice(t, info.LicenseExpressions, []string{"Apache-2.0"}) + assertSlice(t, info.CopyrightNotices, []string{"2024 TOML Author"}) } func TestProject_AllReuseInfo(t *testing.T) { @@ -350,7 +349,7 @@ func TestProject_FakeRepository(t *testing.T) { t.Fatal(err) } - if p.ReuseTOML == nil { + if p.ReuseTOMLs["."] == nil { t.Fatal("expected REUSE.toml") } @@ -387,14 +386,14 @@ func TestProject_FakeRepository(t *testing.T) { assertSlice(t, info.LicenseExpressions, []string{"MIT"}) assertSlice(t, info.CopyrightNotices, []string{"2024 Visible"}) - // assets/logo.png has a .license sidecar, which takes priority. + // The asset annotation overrides the .license sidecar. info, err = p.ReuseInfoOf("assets/logo.png") if err != nil { t.Fatal(err) } assertSlice(t, info.LicenseExpressions, []string{"CC0-1.0"}) - if info.SourceType != DotLicense { - t.Errorf("assets/logo.png SourceType = %v, want DotLicense", info.SourceType) + if info.SourceType != ReuseToml { + t.Errorf("assets/logo.png SourceType = %v, want ReuseToml", info.SourceType) } // License files in LICENSES/ should be discovered. diff --git a/toml/example_test.go b/toml/example_test.go new file mode 100644 index 0000000..8ae2363 --- /dev/null +++ b/toml/example_test.go @@ -0,0 +1,43 @@ +package toml_test + +import ( + "fmt" + + "github.com/git-pkgs/reuse/toml" +) + +func ExampleParseReuseTOML() { + config, err := toml.ParseReuseTOML(`version = 1 +[[annotations]] +path = "src/**" +SPDX-License-Identifier = "MIT" +`) + if err != nil { + fmt.Println(err) + return + } + info, precedence, ok := config.ReuseInfoOf("src/main.go") + if ok { + fmt.Println(info.LicenseExpressions) + fmt.Println(precedence) + } + // Output: + // [MIT] + // closest +} + +func ExampleParseReuseTOMLFile() { + config, err := toml.ParseReuseTOMLFile("../testdata/fake_repository/REUSE.toml") + if err != nil { + fmt.Println(err) + return + } + info, precedence, ok := config.ReuseInfoOf("src/main.go") + if ok { + fmt.Println(info.LicenseExpressions) + fmt.Println(precedence) + } + // Output: + // [MIT] + // closest +} diff --git a/toml/toml.go b/toml/toml.go index 15818af..79c1bb0 100644 --- a/toml/toml.go +++ b/toml/toml.go @@ -3,6 +3,7 @@ package toml import ( "fmt" "os" + "slices" btoml "github.com/BurntSushi/toml" "github.com/git-pkgs/reuse/internal/core" @@ -125,8 +126,8 @@ func (t *ReuseTOML) ReuseInfoOf(path string) (core.ReuseInfo, core.PrecedenceTyp ann := &t.Annotations[i] if ann.Matches(path) { info := core.ReuseInfo{ - LicenseExpressions: ann.Licenses, - CopyrightNotices: ann.Copyrights, + LicenseExpressions: slices.Clone(ann.Licenses), + CopyrightNotices: slices.Clone(ann.Copyrights), SourcePath: t.Source, SourceType: core.ReuseToml, }