Fix REUSE resolution and SPDX extraction

README.md

@@ -1,76 +1,125 @@
 # reuse
 
-Go library for parsing REUSE-compliant projects (spec v3.3). Extracts SPDX license and copyright information from file headers, .license sidecars, REUSE.toml annotations, and .reuse/dep5 files.
+Go library for reading REUSE licensing metadata from file headers, `.license`
+sidecars, `REUSE.toml` annotations, and `.reuse/dep5` files. SPDX expressions
+stay as raw strings; use [spdx](https://github.com/git-pkgs/spdx) to validate
+them. The package does not check whether a project is REUSE compliant.
 
-SPDX expressions are stored as raw strings. Validation is left to consumers (see [github.com/git-pkgs/spdx](https://github.com/git-pkgs/spdx) if you need it).
-
-## Installation
-
-```bash
+```sh
 go get github.com/git-pkgs/reuse
 ```
 
-## Usage
-
-### Parse a project
+## Read a project
 
 ```go
-import "github.com/git-pkgs/reuse"
+import (
+    "fmt"
+    "log"
 
-p, err := reuse.OpenProject("/path/to/repo")
+    "github.com/git-pkgs/reuse"
+)
 
-// Get licensing info for a single file.
-info, err := p.ReuseInfoOf("src/main.go")
-fmt.Println(info.LicenseExpressions) // ["MIT"]
-fmt.Println(info.CopyrightNotices)   // ["2024 Jane Doe <jane@example.com>"]
-
-// Walk all covered files.
-all, err := p.AllReuseInfo()
-for path, info := range all {
-    fmt.Printf("%s: %v\n", path, info.LicenseExpressions)
+project, err := reuse.OpenProject("/path/to/repo")
+if err != nil {
+    log.Fatal(err)
+}
+info, err := project.ReuseInfoOf("src/main.go")
+if err != nil {
+    log.Fatal(err)
 }
+fmt.Println(info.LicenseExpressions)
+fmt.Println(info.CopyrightNotices)
 ```
 
-### Extract SPDX tags from text
+`OpenProject` loads annotation files once. `Project.ReuseTOMLs` maps relative
+directory names to parsed TOML files, with `"."` for the root. Each lookup reads
+the source file or its sidecar; reopen the project after changing annotations.
+Paths passed to `ReuseInfoOf` must be relative to the project root.
 
-```go
-info := reuse.ExtractReuseInfo(`// SPDX-License-Identifier: MIT OR Apache-2.0
-// SPDX-FileCopyrightText: 2024 Alice`)
+A sidecar replaces the source file's header, then participates in annotation
+resolution. The outermost matching `override` suppresses closer annotations
+and both header and sidecar. Otherwise, the nearest `closest` values fill
+missing licence and copyright fields separately. `aggregate` adds values;
+ancestor aggregates still apply above an override. Nested TOML patterns are
+relative to their own directory, and the last matching table within one file
+is used. These rules follow the [REUSE specification](https://reuse.software/spec-3.3/#reuse-toml).
 
-fmt.Println(info.LicenseExpressions) // ["MIT OR Apache-2.0"]
-fmt.Println(info.CopyrightNotices)   // ["2024 Alice"]
-```
+When no TOML files are present, `.reuse/dep5` entries are aggregated with the
+header or sidecar. `SourcePath` and `SourceType` identify the primary source;
+the combined result does not retain provenance for every individual value.
+
+`project.AllReuseInfo()` returns a map for covered files, including empty
+results when no metadata was found. The walk skips symlinks, empty files,
+licence files, sidecars, SPDX documents, and the built-in ignored directories.
+It does not apply `.gitignore` or exclude VCS submodules and Meson subprojects.
+Use individual lookups when the caller has its own file-selection policy.
+
+## Read text or annotation files
 
-### Parse REUSE.toml
+Text extraction is in the `extract` subpackage:
 
 ```go
-rt, err := reuse.ParseReuseTOMLFile("REUSE.toml")
+import "github.com/git-pkgs/reuse/extract"
 
-info, precedence, ok := rt.ReuseInfoOf("src/main.go")
-if ok {
-    fmt.Println(info.LicenseExpressions)
-    fmt.Println(precedence) // "closest", "aggregate", or "override"
-}
+info := extract.ExtractReuseInfo(`// SPDX-License-Identifier: MIT OR Apache-2.0
+// SPDX-FileCopyrightText: 2024 Alice`)
+fmt.Println(info.LicenseExpressions) // [MIT OR Apache-2.0]
 ```
 
-### Parse .reuse/dep5
+`ExtractReuseInfo` requires complete text and treats its final line as
+terminated. Passing a prefix ending at `SPDX-License-Identifier: MIT` can
+therefore lose a later `OR Apache-2.0`. Use `peek` for bounded-prefix claims,
+and keep those observations separate from resolved project metadata.
+
+The extractor handles REUSE ignore blocks and collects snippet declarations
+alongside file declarations. It does not retain snippet scopes or byte spans.
+Empty tags are ignored, and values do not continue onto subsequent lines.
+Returned values own their strings, so retaining a short header does not retain
+the whole input. `extract.ExtractFromFile` reads the complete file and returns
+empty metadata for NUL bytes or invalid UTF-8 in its first 8 KiB, extending that
+probe just enough to finish a crossing code point. UTF-16 is not decoded.
+
+The `toml` subpackage reads annotations without opening a project:
 
 ```go
-d, err := reuse.ParseDep5File(".reuse/dep5")
+import "github.com/git-pkgs/reuse/toml"
 
-info, ok := d.ReuseInfoOf("docs/guide.md")
+config, err := toml.ParseReuseTOMLFile("REUSE.toml")
+if err != nil {
+    log.Fatal(err)
+}
+info, precedence, ok := config.ReuseInfoOf("src/main.go")
 if ok {
-    fmt.Println(info.LicenseExpressions)
+    fmt.Println(info.LicenseExpressions, precedence)
 }
 ```
 
+For DEP5, use `dep5.ParseDep5File` from `github.com/git-pkgs/reuse/dep5`.
+Both subpackages also provide string parsers, `toml.ParseReuseTOML` and
+`dep5.ParseDep5`. Parsed examples are included in the Go tests.
+
+File reads and traversal have no size or file-count limits. `AllReuseInfo`
+retains all results; applications processing large trees can use individual
+lookups to consume results as they go. Concurrent lookups are supported while
+the project and its parsed annotations remain unchanged.
+
 ## Development
 
-```bash
-git clone --recurse-submodules https://github.com/git-pkgs/reuse
-go test -v -race ./...
+Initialize the example fixture after cloning, then run the tests and benchmarks:
+
+```sh
+git submodule update --init
+go test -race ./...
+go test ./extract -run '^$' -bench BenchmarkExtractReuseInfo -benchmem
+go test ./extract -run '^$' -bench BenchmarkRetainedHeaders -benchtime 1x
+go test . -run '^$' -bench BenchmarkProject -benchmem
 ```
 
+The extraction benchmarks cover short and long inputs, ignore blocks, and
+snippets. `BenchmarkRetainedHeaders` reports heap retained after collecting
+short licence values from separate 128 KiB inputs. `BenchmarkProject` includes
+file reads and annotation resolution; its results depend on filesystem caches.
+
 ## License
 
 [MIT](LICENSE).

dep5/example_test.go

@@ -0,0 +1,26 @@
+package dep5_test
+
+import (
+	"fmt"
+
+	"github.com/git-pkgs/reuse/dep5"
+)
+
+func ExampleParseDep5() {
+	config, err := dep5.ParseDep5(`Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+
+Files: docs/*
+Copyright: 2024 Alice
+License: CC-BY-4.0
+`)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	info, ok := config.ReuseInfoOf("docs/guide.md")
+	if ok {
+		fmt.Println(info.LicenseExpressions)
+	}
+	// Output:
+	// [CC-BY-4.0]
+}

example_test.go

@@ -0,0 +1,25 @@
+package reuse_test
+
+import (
+	"fmt"
+
+	"github.com/git-pkgs/reuse"
+)
+
+func ExampleProject_ReuseInfoOf() {
+	project, err := reuse.OpenProject("testdata/fake_repository")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	info, err := project.ReuseInfoOf("src/main.go")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(info.LicenseExpressions)
+	fmt.Println(info.CopyrightNotices)
+	// Output:
+	// [MIT]
+	// [2024 Alice]
+}

extract/benchmark_test.go

@@ -0,0 +1,50 @@
+package extract_test
+
+import (
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/git-pkgs/reuse/extract"
+)
+
+func BenchmarkExtractReuseInfo(b *testing.B) {
+	for _, tc := range []struct {
+		name string
+		text string
+	}{
+		{"plain-1KiB", strings.Repeat("x", 1024)},
+		{"plain-64KiB", strings.Repeat("x", 64<<10)},
+		{"header-1KiB", "// SPDX-License-Identifier: MIT\n" + strings.Repeat("x", 1024)},
+		{"header-64KiB", "// SPDX-License-Identifier: MIT\n" + strings.Repeat("x", 64<<10)},
+		{"ignored", "REUSE-IgnoreStart\nSPDX-License-Identifier: GPL-3.0-only\nREUSE-IgnoreEnd\nSPDX-License-Identifier: MIT\n"},
+		{"snippet", "SPDX-License-Identifier: MIT\nSPDX-SnippetBegin\nSPDX-License-Identifier: Apache-2.0\nSPDX-SnippetEnd\n"},
+	} {
+		b.Run(tc.name, func(b *testing.B) {
+			b.ReportAllocs()
+			b.SetBytes(int64(len(tc.text)))
+			for b.Loop() {
+				_ = extract.ExtractReuseInfo(tc.text)
+			}
+		})
+	}
+}
+
+func BenchmarkRetainedHeaders(b *testing.B) {
+	const count = 64
+	text := "// SPDX-License-Identifier: MIT\n" + strings.Repeat("x", 128<<10)
+	for b.Loop() {
+		values := make([]string, 0, count)
+		var before, after runtime.MemStats
+		runtime.GC()
+		runtime.ReadMemStats(&before)
+		for range count {
+			info := extract.ExtractReuseInfo(strings.Clone(text))
+			values = append(values, info.LicenseExpressions...)
+		}
+		runtime.GC()
+		runtime.ReadMemStats(&after)
+		b.ReportMetric(float64(max(0, int64(after.HeapAlloc)-int64(before.HeapAlloc)))/count, "retained-B/file")
+		runtime.KeepAlive(values)
+	}
+}

extract/example_test.go

@@ -0,0 +1,17 @@
+package extract_test
+
+import (
+	"fmt"
+
+	"github.com/git-pkgs/reuse/extract"
+)
+
+func ExampleExtractReuseInfo() {
+	info := extract.ExtractReuseInfo(`// SPDX-License-Identifier: MIT OR Apache-2.0
+// SPDX-FileCopyrightText: 2024 Alice`)
+	fmt.Println(info.LicenseExpressions)
+	fmt.Println(info.CopyrightNotices)
+	// Output:
+	// [MIT OR Apache-2.0]
+	// [2024 Alice]
+}

extract/extract.go

@@ -12,18 +12,18 @@ import (
 
 var (
 	// Trailing comment-end markers stripped from matched values.
-	endPattern = `[\s]*(?:\*/|-->|"|'>|\]\s*::|"\))?[\s]*$`
+	endPattern = `[ \t\f]*(?:\*/|-->|"|'>|\][ \t\f]*::|"\))?[ \t\f]*$`
 
 	licensePattern = regexp.MustCompile(
-		`(?m)^.*?SPDX-License-Identifier:\s*(?P<value>.*?)` + endPattern,
+		`(?m)SPDX-License-Identifier:[ \t\f]*(?P<value>.*?)` + endPattern,
 	)
 
 	copyrightPattern = regexp.MustCompile(
-		`(?m)^.*?SPDX-(?:File|Snippet)CopyrightText:\s*(?P<value>.*?)` + endPattern,
+		`(?m)SPDX-(?:File|Snippet)CopyrightText:[ \t\f]*(?P<value>.*?)` + endPattern,
 	)
 
 	contributorPattern = regexp.MustCompile(
-		`(?m)^.*?SPDX-FileContributor:\s*(?P<value>.*?)` + endPattern,
+		`(?m)SPDX-FileContributor:[ \t\f]*(?P<value>.*?)` + endPattern,
 	)
 
 	snippetBeginPattern = regexp.MustCompile(`(?m)^.*?SPDX-SnippetBegin`)
@@ -32,14 +32,18 @@ var (
 
 // ExtractReuseInfo extracts SPDX license, copyright, and contributor
 // information from the given text. It handles REUSE-IgnoreStart/End blocks
-// and SPDX-SnippetBegin/End regions.
+// and SPDX-SnippetBegin/End regions. The text must be complete; the final line
+// is treated as terminated. Returned values do not retain the input string.
 func ExtractReuseInfo(text string) core.ReuseInfo {
 	// Normalize line endings.
 	text = strings.ReplaceAll(text, "\r\n", "\n")
 	text = strings.ReplaceAll(text, "\r", "\n")
 
 	// Strip ignore blocks.
 	text = FilterIgnoreBlocks(text)
+	if !strings.Contains(text, "SPDX-") {
+		return core.ReuseInfo{}
+	}
 
 	// Strip snippet regions but keep the snippet tags themselves for extraction.
 	// The spec says snippet info applies only to the snippet, but for file-level
@@ -59,19 +63,19 @@ func extractFromText(text string, info *core.ReuseInfo) {
 	for _, m := range licensePattern.FindAllStringSubmatch(text, -1) {
 		val := strings.TrimSpace(m[1])
 		if val != "" {
-			info.LicenseExpressions = append(info.LicenseExpressions, val)
+			info.LicenseExpressions = append(info.LicenseExpressions, strings.Clone(val))
 		}
 	}
 	for _, m := range copyrightPattern.FindAllStringSubmatch(text, -1) {
 		val := strings.TrimSpace(m[1])
 		if val != "" {
-			info.CopyrightNotices = append(info.CopyrightNotices, val)
+			info.CopyrightNotices = append(info.CopyrightNotices, strings.Clone(val))
 		}
 	}
 	for _, m := range contributorPattern.FindAllStringSubmatch(text, -1) {
 		val := strings.TrimSpace(m[1])
 		if val != "" {
-			info.Contributors = append(info.Contributors, val)
+			info.Contributors = append(info.Contributors, strings.Clone(val))
 		}
 	}
 }
@@ -80,6 +84,9 @@ func extractFromText(text string, info *core.ReuseInfo) {
 // regions and the snippet regions themselves. If no snippets are found, the
 // entire text is returned as the file portion.
 func splitSnippets(text string) (string, []string) {
+	if !strings.Contains(text, "SPDX-SnippetBegin") {
+		return text, nil
+	}
 	beginLocs := snippetBeginPattern.FindAllStringIndex(text, -1)
 	if len(beginLocs) == 0 {
 		return text, nil
@@ -180,7 +187,13 @@ const binaryCheckLimit = 8192
 func isBinary(data []byte) bool {
 	check := data
 	if len(check) > binaryCheckLimit {
-		check = check[:binaryCheckLimit]
+		// Include a code point that straddles the probe boundary.
+		start := binaryCheckLimit - 1
+		for start > binaryCheckLimit-utf8.UTFMax && !utf8.RuneStart(data[start]) {
+			start--
+		}
+		_, size := utf8.DecodeRune(data[start:])
+		check = check[:max(binaryCheckLimit, start+size)]
 	}
 
 	if slices.Contains(check, 0) {