From ab2959cd9ac224f62a51b23ae2a40a0e1631b272 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 29 Jul 2026 03:38:45 -0700 Subject: [PATCH 1/6] Add a reproducible public evidence catalog Record pinned inputs, exact reproduction commands, evidence checksums, claim scope, and limitations for public assurance cases. Add a standard-library verifier and a Make target for checking the full catalog or inspecting one case. --- Makefile | 5 +- internal/tools/publicevidence/main.go | 311 ++++++++ internal/tools/publicevidence/main_test.go | 110 +++ test/evidence/README.md | 39 + test/evidence/cases.json | 865 +++++++++++++++++++++ 5 files changed, 1329 insertions(+), 1 deletion(-) create mode 100644 internal/tools/publicevidence/main.go create mode 100644 internal/tools/publicevidence/main_test.go create mode 100644 test/evidence/README.md create mode 100644 test/evidence/cases.json diff --git a/Makefile b/Makefile index 8b294193..f47ff0d7 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ EXE_SUFFIX=$(if $(filter Windows_NT,$(OS)),.exe,) GOLANGCI_LINT=$(GOPATH_BIN)/golangci-lint$(EXE_SUFFIX) FUZZTIME?=60s -.PHONY: build build-full build-lite fmt fmt-check lint install-hooks test fuzz run generate docs-config docs-schema docs-schema-md docs-support-matrix docs-components smoke benchmark benchmark-samples benchmark-report licenses +.PHONY: build build-full build-lite fmt fmt-check lint install-hooks test fuzz run generate docs-config docs-schema docs-schema-md docs-support-matrix docs-components smoke evidence benchmark benchmark-samples benchmark-report licenses build: build-full build-lite @@ -41,6 +41,9 @@ fuzz: smoke: go test -tags "smoke" ./test/smoke/ -v -count=1 -timeout 15m $(if $(ARGS),$(ARGS),) +evidence: + go run ./internal/tools/publicevidence $(if $(CASE),-case $(CASE),) + benchmark: build-full bin/$(BINARY_NAME)$(EXE_SUFFIX) benchmark $(if $(ARGS),$(ARGS),) diff --git a/internal/tools/publicevidence/main.go b/internal/tools/publicevidence/main.go new file mode 100644 index 00000000..f5e77d3c --- /dev/null +++ b/internal/tools/publicevidence/main.go @@ -0,0 +1,311 @@ +// Command publicevidence validates and displays Bomly's public evidence +// catalog. +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +const ( + catalogSchema = "bomly.public-evidence/v1" + defaultCatalog = "test/evidence/cases.json" +) + +var ( + caseIDPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) + revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) +) + +type catalog struct { + SchemaVersion string `json:"schema_version"` + Cases []evidenceCase `json:"cases"` +} + +type evidenceCase struct { + ID string `json:"id"` + Title string `json:"title"` + Area string `json:"area"` + EvidenceLevel string `json:"evidence_level"` + Inputs []input `json:"inputs"` + RequiredTools []string `json:"required_tools,omitempty"` + Reproduce [][]string `json:"reproduce"` + Evidence []artifact `json:"evidence"` + Proves []string `json:"proves"` + Limitations []string `json:"limitations"` +} + +type input struct { + Kind string `json:"kind"` + Location string `json:"location"` + Ref string `json:"ref,omitempty"` + Revision string `json:"revision,omitempty"` + SHA256 string `json:"sha256,omitempty"` +} + +type artifact struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` +} + +func main() { + catalogPath := flag.String("catalog", defaultCatalog, "path to the public evidence catalog") + caseID := flag.String("case", "", "show one evidence case") + flag.Parse() + + root, err := repositoryRoot() + if err != nil { + exitError(err) + } + loaded, err := loadCatalog(filepath.Join(root, filepath.FromSlash(*catalogPath))) + if err != nil { + exitError(err) + } + if err := validateCatalog(root, loaded); err != nil { + exitError(err) + } + + selected := loaded.Cases + if *caseID != "" { + selected = nil + for _, current := range loaded.Cases { + if current.ID == *caseID { + selected = []evidenceCase{current} + break + } + } + if len(selected) == 0 { + exitError(fmt.Errorf("unknown evidence case %q", *caseID)) + } + } + printCases(selected) +} + +func repositoryRoot() (string, error) { + current, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("resolve working directory: %w", err) + } + for { + if info, statErr := os.Stat(filepath.Join(current, "go.mod")); statErr == nil && !info.IsDir() { + return current, nil + } + parent := filepath.Dir(current) + if parent == current { + return "", errors.New("find repository root: go.mod not found") + } + current = parent + } +} + +func loadCatalog(path string) (catalog, error) { + file, err := os.Open(path) + if err != nil { + return catalog{}, fmt.Errorf("open evidence catalog: %w", err) + } + defer file.Close() + + decoder := json.NewDecoder(io.LimitReader(file, 2<<20)) + decoder.DisallowUnknownFields() + var loaded catalog + if err := decoder.Decode(&loaded); err != nil { + return catalog{}, fmt.Errorf("decode evidence catalog: %w", err) + } + if err := ensureJSONEOF(decoder); err != nil { + return catalog{}, err + } + return loaded, nil +} + +func ensureJSONEOF(decoder *json.Decoder) error { + var extra any + err := decoder.Decode(&extra) + if errors.Is(err, io.EOF) { + return nil + } + if err == nil { + return errors.New("decode evidence catalog: multiple JSON values") + } + return fmt.Errorf("decode evidence catalog trailing data: %w", err) +} + +func validateCatalog(root string, loaded catalog) error { + if loaded.SchemaVersion != catalogSchema { + return fmt.Errorf("unsupported evidence catalog schema %q", loaded.SchemaVersion) + } + if len(loaded.Cases) == 0 { + return errors.New("evidence catalog contains no cases") + } + seen := make(map[string]struct{}, len(loaded.Cases)) + previous := "" + for index, current := range loaded.Cases { + if !caseIDPattern.MatchString(current.ID) { + return fmt.Errorf("case %d has invalid id %q", index+1, current.ID) + } + if _, exists := seen[current.ID]; exists { + return fmt.Errorf("duplicate evidence case %q", current.ID) + } + seen[current.ID] = struct{}{} + if previous != "" && current.ID < previous { + return fmt.Errorf("evidence cases are not sorted: %q follows %q", current.ID, previous) + } + previous = current.ID + } + for _, current := range loaded.Cases { + if err := validateCase(root, current); err != nil { + return fmt.Errorf("case %q: %w", current.ID, err) + } + } + return nil +} + +func validateCase(root string, current evidenceCase) error { + if strings.TrimSpace(current.Title) == "" || strings.TrimSpace(current.Area) == "" { + return errors.New("title and area are required") + } + switch current.EvidenceLevel { + case "deterministic", "pinned-input", "live-service", "manual-assurance": + default: + return fmt.Errorf("unsupported evidence level %q", current.EvidenceLevel) + } + if len(current.Inputs) == 0 { + return errors.New("at least one input is required") + } + for _, item := range current.Inputs { + if err := validateInput(root, item); err != nil { + return err + } + } + if len(current.Reproduce) == 0 { + return errors.New("at least one reproduction command is required") + } + for index, command := range current.Reproduce { + if len(command) == 0 { + return fmt.Errorf("reproduction command %d is empty", index+1) + } + for _, argument := range command { + if argument == "" { + return fmt.Errorf("reproduction command %d contains an empty argument", index+1) + } + } + } + if len(current.Evidence) == 0 { + return errors.New("at least one evidence artifact is required") + } + for _, item := range current.Evidence { + if err := validateArtifact(root, item); err != nil { + return err + } + } + if len(current.Proves) == 0 || len(current.Limitations) == 0 { + return errors.New("proves and limitations must both be explicit") + } + return nil +} + +func validateInput(root string, current input) error { + if strings.TrimSpace(current.Location) == "" { + return errors.New("input location is required") + } + switch current.Kind { + case "git": + if !revisionPattern.MatchString(current.Revision) { + return errors.New("git input requires a full lowercase commit revision") + } + case "fixture": + if !hashPattern.MatchString(current.SHA256) { + return errors.New("fixture input requires a SHA-256 hash") + } + return validateArtifact(root, artifact{Path: current.Location, SHA256: current.SHA256}) + case "container": + if current.Ref == "" { + return errors.New("container input requires an image reference") + } + case "workflow": + if !hashPattern.MatchString(current.SHA256) { + return errors.New("workflow input requires a SHA-256 hash") + } + return validateArtifact(root, artifact{Path: current.Location, SHA256: current.SHA256}) + default: + return fmt.Errorf("unsupported input kind %q", current.Kind) + } + return nil +} + +func validateArtifact(root string, item artifact) error { + if !hashPattern.MatchString(item.SHA256) { + return fmt.Errorf("artifact %q has an invalid SHA-256 hash", item.Path) + } + clean := filepath.Clean(filepath.FromSlash(item.Path)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return fmt.Errorf("artifact path %q must stay inside the repository", item.Path) + } + path := filepath.Join(root, clean) + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("inspect artifact %q: %w", item.Path, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("artifact %q is not a regular file", item.Path) + } + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read artifact %q: %w", item.Path, err) + } + sum := sha256.Sum256(data) + actual := hex.EncodeToString(sum[:]) + if actual != item.SHA256 { + return fmt.Errorf("artifact %q hash is %s, want %s", item.Path, actual, item.SHA256) + } + return nil +} + +func printCases(cases []evidenceCase) { + sort.Slice(cases, func(i, j int) bool { return cases[i].ID < cases[j].ID }) + fmt.Printf("Verified %d public evidence case(s).\n", len(cases)) + for _, current := range cases { + fmt.Printf("\n%s — %s\n", current.ID, current.Title) + fmt.Printf(" Area: %s; evidence: %s\n", current.Area, current.EvidenceLevel) + for _, command := range current.Reproduce { + fmt.Printf(" Reproduce: %s\n", shellCommand(command)) + } + for _, limitation := range current.Limitations { + fmt.Printf(" Limitation: %s\n", limitation) + } + } +} + +func shellCommand(command []string) string { + quoted := make([]string, len(command)) + for index, argument := range command { + if argument != "" && strings.IndexFunc(argument, func(r rune) bool { + if (r >= 'a' && r <= 'z') || + (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') { + return false + } + return !strings.ContainsRune("@%_+=:,./-", r) + }) == -1 { + quoted[index] = argument + continue + } + quoted[index] = "'" + strings.ReplaceAll(argument, "'", "'\"'\"'") + "'" + } + return strings.Join(quoted, " ") +} + +func exitError(err error) { + fmt.Fprintln(os.Stderr, "public evidence:", err) + os.Exit(1) +} diff --git a/internal/tools/publicevidence/main_test.go b/internal/tools/publicevidence/main_test.go new file mode 100644 index 00000000..f0869479 --- /dev/null +++ b/internal/tools/publicevidence/main_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadAndValidateCatalog(t *testing.T) { + root := t.TempDir() + artifactPath := filepath.Join(root, "result.json") + data := []byte(`{"ok":true}`) + if err := os.WriteFile(artifactPath, data, 0o600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + hash := hex.EncodeToString(sum[:]) + catalogPath := filepath.Join(root, "cases.json") + document := `{ + "schema_version": "bomly.public-evidence/v1", + "cases": [{ + "id": "example-case", + "title": "Example", + "area": "graph", + "evidence_level": "deterministic", + "inputs": [{"kind": "fixture", "location": "result.json", "sha256": "` + hash + `"}], + "reproduce": [["go", "test", "./..."]], + "evidence": [{"path": "result.json", "sha256": "` + hash + `"}], + "proves": ["The example succeeds."], + "limitations": ["The example covers one input."] + }] +}` + if err := os.WriteFile(catalogPath, []byte(document), 0o600); err != nil { + t.Fatal(err) + } + + loaded, err := loadCatalog(catalogPath) + if err != nil { + t.Fatal(err) + } + if err := validateCatalog(root, loaded); err != nil { + t.Fatal(err) + } +} + +func TestValidateCatalogRejectsUnpinnedGitAndChangedArtifact(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "result.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + current := catalog{ + SchemaVersion: catalogSchema, + Cases: []evidenceCase{{ + ID: "git-case", + Title: "Git case", + Area: "graph", + EvidenceLevel: "pinned-input", + Inputs: []input{{ + Kind: "git", + Location: "https://github.com/example/project", + Revision: "main", + }}, + Reproduce: [][]string{{"go", "test", "./..."}}, + Evidence: []artifact{{Path: "result.json", SHA256: strings.Repeat("0", 64)}}, + Proves: []string{"A result."}, + Limitations: []string{"One input."}, + }}, + } + if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "full lowercase commit") { + t.Fatalf("validateCatalog() error = %v", err) + } + + current.Cases[0].Inputs[0].Revision = strings.Repeat("a", 40) + if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "artifact") { + t.Fatalf("validateCatalog() error = %v", err) + } +} + +func TestValidateCatalogRejectsUnsortedAndUnknownFields(t *testing.T) { + root := t.TempDir() + current := catalog{ + SchemaVersion: catalogSchema, + Cases: []evidenceCase{ + {ID: "z-case"}, + {ID: "a-case"}, + }, + } + if err := validateCatalog(root, current); err == nil || !strings.Contains(err.Error(), "not sorted") { + t.Fatalf("validateCatalog() error = %v", err) + } + + path := filepath.Join(root, "cases.json") + if err := os.WriteFile(path, []byte(`{"schema_version":"bomly.public-evidence/v1","cases":[],"extra":true}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := loadCatalog(path); err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("loadCatalog() error = %v", err) + } +} + +func TestShellCommandQuotesPatterns(t *testing.T) { + got := shellCommand([]string{"go", "test", "-run", "TestScan$/scan-npm$"}) + want := "go test -run 'TestScan$/scan-npm$'" + if got != want { + t.Fatalf("shellCommand() = %q, want %q", got, want) + } +} diff --git a/test/evidence/README.md b/test/evidence/README.md new file mode 100644 index 00000000..2fca08af --- /dev/null +++ b/test/evidence/README.md @@ -0,0 +1,39 @@ +# Public evidence catalog + +This directory maps Bomly claims to repeatable tests and checked-in evidence. +It is not a second test suite. The catalog points to the smoke tests, focused +unit tests, fixtures, and manually started assurance workflows that already +own each behavior. + +Run the catalog check from the repository root: + +```sh +make evidence +``` + +Show one case and its exact reproduction command: + +```sh +make evidence CASE=graph-npm +``` + +The checker verifies that: + +- every remote Git input has a full commit revision; +- every fixture, workflow, and result file has the recorded SHA-256 checksum; +- every case says what it proves and what it does not prove; +- case IDs are unique and stable. + +## Evidence levels + +- `deterministic` uses checked-in inputs or local services and should produce + the same normalized result. +- `pinned-input` uses a public repository revision. Package-manager tools or + registries can still affect build-tool-backed resolution. +- `live-service` combines a pinned project with current advisory data. It is a + dated observation because advisory services change. +- `manual-assurance` starts a GitHub Actions workflow and stores the detailed + run report as a workflow artifact. + +The catalog schema belongs only to repository assurance metadata. It does not +change Bomly's CLI, MCP, SDK, or plugin schemas. diff --git a/test/evidence/cases.json b/test/evidence/cases.json new file mode 100644 index 00000000..272bcaa2 --- /dev/null +++ b/test/evidence/cases.json @@ -0,0 +1,865 @@ +{ + "schema_version": "bomly.public-evidence/v1", + "cases": [ + { + "id": "baseline-policy", + "title": "Finding baseline lifecycle", + "area": "policy", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestFindingBaselineWorkflow$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/finding-baseline-workflow.golden.json", + "sha256": "8fa2b220af563a89d1fbaaab03a75ccc0952ae43251438bc27a2314301315aa0" + } + ], + "proves": [ + "A project baseline can keep a denied package finding visible with suppressed policy status." + ], + "limitations": [ + "The local matcher fixture is deterministic but does not measure advisory freshness." + ] + }, + { + "id": "container-inventory", + "title": "Container package inventory", + "area": "targets", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "container", + "location": "Docker Hub", + "ref": "alpine:3.20" + } + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestContainerScan/container-scan-alpine$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/container-scan-alpine.golden.json", + "sha256": "50f7d4426ead3af9ea5a07df7eeb77743e7aa1412260e1a665a7301f3c22ecc5" + } + ], + "proves": [ + "Bomly can inventory the operating-system packages in the checked Alpine image." + ], + "limitations": [ + "The upstream image tag can move; the checked-in golden is a snapshot, not an immutable image claim." + ] + }, + { + "id": "graph-bun", + "title": "Bun lockfile graph", + "area": "dependency-graph", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-bun", + "ref": "v1.0.0", + "revision": "358a2a920fe9c2f7e596c514f36df1c30c1ab185" + } + ], + "required_tools": [ + "git" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-bun$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-bun.golden.json", + "sha256": "c61ccae15f3d36028ec1e741436abbf036236b9e2f018393b863e98d14360fb4" + } + ], + "proves": [ + "The native Bun detector preserves the package inventory and dependency placement represented by the pinned lockfile." + ], + "limitations": [ + "This is one Bun lockfile shape and does not prove behavior for every historical Bun format." + ] + }, + { + "id": "graph-go", + "title": "Go module graph", + "area": "dependency-graph", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-go$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-go.golden.json", + "sha256": "728f949898446657c4987bc1202a8d00c78b118868b4945141e4db8c4c54cdea" + } + ], + "proves": [ + "The Go detector resolves a build-tool-backed module graph for the pinned repository." + ], + "limitations": [ + "The result depends on a compatible Go toolchain and the evidence available from that toolchain." + ] + }, + { + "id": "graph-maven", + "title": "Maven dependency graph", + "area": "dependency-graph", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-java-maven", + "ref": "v1.0.0", + "revision": "93bb3aae614e2f2c6cb65f5ea2315846f5234150" + } + ], + "required_tools": [ + "git", + "java", + "mvn" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-maven$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-maven.golden.json", + "sha256": "a1077544af4bd637311e1767abe2a140e3ba8f0f65a64e662378e5ccca07de9b" + } + ], + "proves": [ + "The Maven detector resolves a build-tool-backed dependency graph for the pinned repository." + ], + "limitations": [ + "Artifact resolution depends on Maven repositories and a compatible Java and Maven installation." + ] + }, + { + "id": "graph-npm", + "title": "npm lockfile graph", + "area": "dependency-graph", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-npm", + "ref": "v1.0.0", + "revision": "559a762aeef68b0e5c818f62dfba67abc369912f" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-npm$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-npm.golden.json", + "sha256": "da252e623cdeaf88db7874f528fd9a2f204c1136c4ab0bb26e4faa943dee79c1" + } + ], + "proves": [ + "The npm detector preserves lockfile package inventory, versions, scopes, and dependency placement." + ], + "limitations": [ + "This case covers the lockfile versions present in one pinned repository." + ] + }, + { + "id": "graph-pnpm", + "title": "pnpm lockfile graph", + "area": "dependency-graph", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-pnpm", + "ref": "v1.0.0", + "revision": "f1b0959f916dfb91db70c54a75b15e5b7f3d16af" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-pnpm$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-pnpm.golden.json", + "sha256": "07991083b233acb7bc856e41275517a30cd5793c4b1afb94a467502374545467" + } + ], + "proves": [ + "The pnpm detector preserves lockfile package inventory and dependency placement." + ], + "limitations": [ + "This case does not cover every pnpm lockfile generation." + ] + }, + { + "id": "graph-python", + "title": "Python requirements graph", + "area": "dependency-graph", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-python-pip", + "revision": "fe04c758134b95dab102e1fce10275f7d18c0cf2" + } + ], + "required_tools": [ + "git", + "pip" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-python-pip$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-python-pip.golden.json", + "sha256": "025d14ad1a3c93f3af22a4a83a358ee85b42d782f50798b285048e35e8632bf2" + } + ], + "proves": [ + "The pip detector reads the pinned requirements lock and preserves the resolved Python package graph." + ], + "limitations": [ + "Unpinned requirements and environment inspection have different fidelity and are not proven by this case." + ] + }, + { + "id": "graph-yarn", + "title": "Yarn lockfile graph", + "area": "dependency-graph", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-yarn", + "ref": "v1.0.0", + "revision": "c84017b43bf0f6ea74281f4174a0c89b88b8cddf" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-yarn$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-yarn.golden.json", + "sha256": "707e1ab208812a53527cc1de888d3e76c410d9b963b49c14076f023ceb5f7b90" + } + ], + "proves": [ + "The Yarn detector preserves package inventory and dependency placement represented by the pinned lockfile." + ], + "limitations": [ + "This case covers one Yarn lockfile family; fallback behavior is documented separately." + ] + }, + { + "id": "performance-stability", + "title": "Repeated cold and warm scan measurements", + "area": "operations", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "fixture", + "location": "test/smoke/testdata/sboms/go.spdx.json", + "sha256": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "make", + "benchmark-samples" + ] + ], + "evidence": [ + { + "path": "internal/tools/benchmarkrun/main.go", + "sha256": "d99c52b52065a95b0cda323f082bbdcb04276ab7cef96f21678550154f340bef" + } + ], + "proves": [ + "The runner records five isolated cold and five shared-cache warm samples with output hashes, timing, memory, and dispersion." + ], + "limitations": [ + "Wall time and memory are machine-specific observations and are not fixed pass-or-fail limits." + ] + }, + { + "id": "portable-platforms", + "title": "Repeated unit tests and release builds", + "area": "operations", + "evidence_level": "manual-assurance", + "inputs": [ + { + "kind": "workflow", + "location": ".github/workflows/portable-assurance.yml", + "sha256": "fadc2ff37a55c3b0dbaed91d3db67745b32b6ca517075e55734cea49a231b4d2" + } + ], + "required_tools": [ + "gh" + ], + "reproduce": [ + [ + "gh", + "workflow", + "run", + "portable-assurance.yml" + ] + ], + "evidence": [ + { + "path": ".github/workflows/portable-assurance.yml", + "sha256": "fadc2ff37a55c3b0dbaed91d3db67745b32b6ca517075e55734cea49a231b4d2" + } + ], + "proves": [ + "The manual workflow repeats Go unit tests on Linux, macOS, and Windows and cross-builds every release target." + ], + "limitations": [ + "This workflow runs unit tests and builds; it does not run smoke tests against remote repositories or services." + ] + }, + { + "id": "reachability-go", + "title": "Go vulnerability reachability", + "area": "reachability", + "evidence_level": "live-service", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-go-gomod", + "ref": "v1.0.0", + "revision": "0f2103c7e671653e519cf5edb0d3e86020202ecf" + } + ], + "required_tools": [ + "git", + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-go-reachability$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-go-reachability.golden.json", + "sha256": "2d1db6318d75c491b8866ff9643ae80635702591046e4be89c362ca1b828a4cd" + } + ], + "proves": [ + "The Go analyzer attaches named analyzer, tier, status, and reason evidence to matched vulnerabilities." + ], + "limitations": [ + "Advisories come from live services and may change; an unreachable result is not proof that a package is safe." + ] + }, + { + "id": "reachability-java", + "title": "Java package reachability", + "area": "reachability", + "evidence_level": "live-service", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-java-maven", + "ref": "v1.0.0", + "revision": "93bb3aae614e2f2c6cb65f5ea2315846f5234150" + } + ], + "required_tools": [ + "git", + "java", + "mvn" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-java-maven-reachability$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-java-maven-reachability.golden.json", + "sha256": "b3fa7485ebcee27a4836f7ad6b318ca53c238b24fa303059dd518007fc8ecb0f" + } + ], + "proves": [ + "The Java analyzer separates package-reachable and package-unreachable vulnerability evidence." + ], + "limitations": [ + "This is package-tier evidence, not symbol-level proof, and live advisory results may change." + ] + }, + { + "id": "reachability-node", + "title": "JavaScript package reachability", + "area": "reachability", + "evidence_level": "live-service", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-npm", + "ref": "v1.0.0", + "revision": "559a762aeef68b0e5c818f62dfba67abc369912f" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-npm-reachability$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-npm-reachability.golden.json", + "sha256": "2bb379a7e0b301357b24a8a1a169b805461530dac392af1908525e3ce5366760" + } + ], + "proves": [ + "The JavaScript analyzer separates package-reachable and package-unreachable vulnerability evidence." + ], + "limitations": [ + "This is package-tier evidence, dynamic loading can reduce confidence, and live advisory results may change." + ] + }, + { + "id": "reachability-python", + "title": "Python package reachability", + "area": "reachability", + "evidence_level": "live-service", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-python-pip", + "revision": "fe04c758134b95dab102e1fce10275f7d18c0cf2" + } + ], + "required_tools": [ + "git", + "pip" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-python-pip-reachability$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-python-pip-reachability.golden.json", + "sha256": "4fbf082566a5562308edd71fd86f155e3eaf1fc7ed0f6dc96c0f2f7778ca3f90" + } + ], + "proves": [ + "The Python analyzer separates package-reachable and package-unreachable vulnerability evidence." + ], + "limitations": [ + "This is package-tier evidence, reflective imports can reduce confidence, and live advisory results may change." + ] + }, + { + "id": "remediation-read-only", + "title": "Canonical read-only remediation guidance", + "area": "remediation", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "fixture", + "location": "internal/remediation/derive_test.go", + "sha256": "4ab715423343aad70cff5ea4a406b642eb687f2552afb894a9edfba4244c7582" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/remediation", + "-run", + "TestDerivePackageRemediation|TestDeriveBuildsCanonicalOccurrenceSuggestions", + "-count=1" + ] + ], + "evidence": [ + { + "path": "internal/remediation/derive_test.go", + "sha256": "4ab715423343aad70cff5ea4a406b642eb687f2552afb894a9edfba4244c7582" + } + ], + "proves": [ + "One central component derives fix status, recommended versions, occurrence actions, and detector advice without applying changes." + ], + "limitations": [ + "Suggestions are read-only evidence and are not guaranteed to work in every checkout." + ] + }, + { + "id": "sbom-cyclonedx-ingest", + "title": "CycloneDX 1.6 ingestion", + "area": "sbom", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "fixture", + "location": "test/smoke/testdata/sboms/go.cdx.json", + "sha256": "78454a7207f98ba0c8d35df8a257f9185c313d22691cc90a06f11809021bed93" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-sbom-cyclonedx$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-sbom-cyclonedx.golden.json", + "sha256": "54f5e33a47d884e1705d22102dd4745b4474295d0cc56da827bc3f172792dabb" + } + ], + "proves": [ + "Bomly ingests the checked CycloneDX 1.6 dependency graph." + ], + "limitations": [ + "The fixture covers one document shape and does not imply lossless conversion from every producer." + ] + }, + { + "id": "sbom-interoperability", + "title": "External SBOM validation", + "area": "sbom", + "evidence_level": "manual-assurance", + "inputs": [ + { + "kind": "workflow", + "location": ".github/workflows/sbom-interoperability.yml", + "sha256": "5434c3b2971034416cacbde5788320d47f3c1e976bdfdc60ff1f077b2e16ef09" + } + ], + "required_tools": [ + "gh" + ], + "reproduce": [ + [ + "gh", + "workflow", + "run", + "sbom-interoperability.yml" + ] + ], + "evidence": [ + { + "path": ".github/workflows/sbom-interoperability.yml", + "sha256": "5434c3b2971034416cacbde5788320d47f3c1e976bdfdc60ff1f077b2e16ef09" + } + ], + "proves": [ + "Checksum-pinned official SPDX and CycloneDX validators can check Bomly's generated canonical SBOMs." + ], + "limitations": [ + "The workflow validates canonical fixtures, not every possible receiving tool or document conversion." + ] + }, + { + "id": "sbom-spdx-ingest", + "title": "SPDX 2.3 ingestion", + "area": "sbom", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "fixture", + "location": "test/smoke/testdata/sboms/go.spdx.json", + "sha256": "99bd846daec887cfcb61d6d5384ccff91c6045482652f4719fab8b312192fe0e" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestScan$/scan-sbom-spdx$" + ] + ], + "evidence": [ + { + "path": "test/smoke/testdata/golden/scan-sbom-spdx.golden.json", + "sha256": "d49e4fc3f9c61fdb4c533116d9a5988e1841c7ccacc3675b93fb069d5ec14a00" + } + ], + "proves": [ + "Bomly ingests the checked SPDX 2.3 dependency graph." + ], + "limitations": [ + "The fixture covers one document shape and does not imply lossless conversion from every producer." + ] + }, + { + "id": "source-change-policy", + "title": "Registry-to-Git source-change review", + "area": "policy", + "evidence_level": "pinned-input", + "inputs": [ + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-npm", + "ref": "assurance/dependency-source-registry-v1", + "revision": "f6127099dad7f8b6fbaa7ed1ceb8e7b0d1e8c864" + }, + { + "kind": "git", + "location": "https://github.com/bomly-dev/example-javascript-npm", + "ref": "assurance/dependency-source-git-v1", + "revision": "96ffda21548628ce36201741b56164ca5c4405b6" + } + ], + "required_tools": [ + "git", + "npm" + ], + "reproduce": [ + [ + "go", + "test", + "-tags", + "smoke", + "./test/smoke/", + "-v", + "-count=1", + "-timeout", + "15m", + "-run", + "TestDependencyDetailRiskPolicy$" + ] + ], + "evidence": [ + { + "path": "test/smoke/audit_test.go", + "sha256": "363c2ab0ecd4bec931c18abb32706c528e22f0dd3331db2691822851d5eb5ae2" + } + ], + "proves": [ + "Diff marks a same-version registry-to-Git move for review and the package auditor can fail it when source-change policy is enabled." + ], + "limitations": [ + "A source change is a review signal, not proof of malicious intent, and unknown source evidence cannot be classified." + ] + } + ] +} From 7dcbb6c72d1181e513aecd8cea423d9b133598ea Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 29 Jul 2026 03:40:00 -0700 Subject: [PATCH 2/6] Document dependency graph evidence Explain the pinned npm, pnpm, Yarn, Bun, Go, Python, and Maven graph cases, including detector fidelity and limitations. Add a deterministic fallback case that proves degraded coverage remains visible instead of being treated as equivalent native evidence. --- docs/evidence/DEPENDENCY_GRAPHS.md | 86 ++++++++++++++++++++++++++++++ test/evidence/cases.json | 38 +++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 docs/evidence/DEPENDENCY_GRAPHS.md diff --git a/docs/evidence/DEPENDENCY_GRAPHS.md b/docs/evidence/DEPENDENCY_GRAPHS.md new file mode 100644 index 00000000..7c28eab5 --- /dev/null +++ b/docs/evidence/DEPENDENCY_GRAPHS.md @@ -0,0 +1,86 @@ +# Dependency graph evidence + +Bomly's findings depend on the graph produced before enrichment or audit. +These public cases check the graph first: package identity, version, +relationship, scope, source, and the manifest that owns each occurrence. + +The cases use complete public example repositories at recorded Git commits. +Their normalized JSON results are checked into `test/smoke/testdata/golden`. +The evidence catalog records a SHA-256 checksum for every result, so an +expected change must also update the public evidence record. + +## What is covered + +| Case | Input evidence | Detector path | What the case checks | +| --- | --- | --- | --- | +| `graph-npm` | `package-lock.json` | npm lockfile detector | Package inventory, versions, scopes, and placement | +| `graph-pnpm` | `pnpm-lock.yaml` | pnpm lockfile detector | Package inventory and placement | +| `graph-yarn` | `yarn.lock` | Yarn lockfile detector | Package inventory and placement | +| `graph-bun` | `bun.lock` | Native Bun detector | Package inventory and placement | +| `graph-go` | `go.mod` and Go tool output | Go detector | Build-tool-backed module graph | +| `graph-python` | Pinned requirements lock | pip detector | Resolved Python package graph | +| `graph-maven` | `pom.xml` and Maven output | Maven detector | Build-tool-backed JVM graph | + +This is placement evidence, not just a list of package names. The Node cases +retain duplicate versions as separate package identities and keep occurrences +attached to the dependency paths represented by each lockfile. + +## Reproduce a case + +From a Bomly CLI checkout: + +```sh +make evidence CASE=graph-npm +``` + +The command verifies the catalog and prints the exact focused smoke command: + +```sh +go test -tags smoke ./test/smoke/ -v -count=1 -timeout 15m \ + -run 'TestScan$/scan-npm$' +``` + +The smoke test builds Bomly, clones the recorded public input, selects the +named detector, normalizes only documented volatile fields, and compares the +result with the checked-in JSON. + +## Example review workflow + +A maintainer updates a lockfile and wants to confirm that Bomly still sees the +same dependency structure: + +1. Run the matching graph case before changing the detector. +2. Make the detector change. +3. Run the case again. +4. Review every golden change as a package, relationship, scope, source, or + manifest-placement change. +5. Update the catalog checksum only when the new graph is intentional. + +This makes a changed count a starting point for review, not the conclusion. +The actual package and occurrence records explain what changed. + +## Degraded and unknown evidence + +Bomly can fall back when a preferred detector cannot run. A fallback is not +silently treated as equivalent: the result records which detector failed, +which detector ran, and whether coverage was reduced. Unresolved package +placement remains `unknown`; a synthetic link to a manifest helps navigation +but does not become a direct or transitive parent claim. + +The graph cases above pin their detector selector. If that detector is not +ready or fails, the smoke test fails instead of accepting a differently shaped +fallback result. + +## Limits + +- A case proves the checked repository and lockfile shape, not every format + version ever produced by that package manager. +- Build-tool-backed cases depend on compatible local tools and may need + registry access to resolve artifacts. +- A fallback inventory can contain useful packages while carrying less + relationship or source evidence than a native graph. +- Project roots, workspaces, local paths, Git dependencies, and arbitrary URL + dependencies stay in the graph but are not queried as published registry + packages. +- Graph evidence does not prove advisory accuracy. Matching is a later step + with separate evidence and limits. diff --git a/test/evidence/cases.json b/test/evidence/cases.json index 272bcaa2..ca2ebabc 100644 --- a/test/evidence/cases.json +++ b/test/evidence/cases.json @@ -86,6 +86,44 @@ "The upstream image tag can move; the checked-in golden is a snapshot, not an immutable image claim." ] }, + { + "id": "degraded-detector-fallback", + "title": "Visible detector fallback", + "area": "dependency-graph", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "fixture", + "location": "internal/engine/pipeline_fallback_test.go", + "sha256": "4c2780f5c41b440b6c51eef05128a5079c33c9b08858fd7722b52febc05fe3c0" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/engine", + "-run", + "TestPipeline_RunRecordsFallbackWarning|TestPipeline_Run_TypesFallbackWarningsAsDegradedCoverage", + "-count=1" + ] + ], + "evidence": [ + { + "path": "internal/engine/pipeline_fallback_test.go", + "sha256": "4c2780f5c41b440b6c51eef05128a5079c33c9b08858fd7722b52febc05fe3c0" + } + ], + "proves": [ + "A detector fallback preserves its origin and produces a typed degraded-coverage warning." + ], + "limitations": [ + "The synthetic detector case proves orchestration behavior, not the fidelity of every concrete fallback." + ] + }, { "id": "graph-bun", "title": "Bun lockfile graph", From 92bcea39b3a2462acb3bac0be9e2c5496fff2592 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 29 Jul 2026 03:41:19 -0700 Subject: [PATCH 3/6] Document policy and vulnerability guidance evidence Publish focused evidence for complex SPDX policy, vulnerability constraints, baselines, risky source changes, persisted findings, analyzer-tier reachability, and canonical read-only remediation guidance. State the limits of live advisory data and unknown evidence explicitly. --- docs/evidence/POLICY_AND_GUIDANCE.md | 117 +++++++++++++++++++++++++++ test/evidence/cases.json | 114 ++++++++++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 docs/evidence/POLICY_AND_GUIDANCE.md diff --git a/docs/evidence/POLICY_AND_GUIDANCE.md b/docs/evidence/POLICY_AND_GUIDANCE.md new file mode 100644 index 00000000..c9859666 --- /dev/null +++ b/docs/evidence/POLICY_AND_GUIDANCE.md @@ -0,0 +1,117 @@ +# Policy and vulnerability-guidance evidence + +Bomly separates observed evidence from policy decisions: + +- enrichment attaches vulnerability and license information to packages; +- analysis can add reachability evidence; +- audit evaluates the available evidence against the selected policy; +- read-only remediation guidance is derived from vulnerability and dependency + evidence during enrichment. + +That separation matters when reviewing results. A vulnerability can be +present but warning-only, suppressed by a baseline, or failing policy. +Reachability is another property of the same vulnerability; it does not +replace severity or policy status. + +## Policy cases + +| Case | What it checks | Expected outcome | +| --- | --- | --- | +| `vulnerability-policy` | Severity, reachability, known exploitation, advisory allowlists, and repeated constraints | Only findings that satisfy the selected constraints fail | +| `license-policy` | SPDX `AND`, `OR`, nesting, exceptions, custom references, and invalid expressions | Valid expressions keep their Boolean meaning; invalid expressions remain visible and fail | +| `baseline-policy` | Create and automatically use a project finding baseline | The finding remains present with suppressed policy status | +| `source-change-policy` | Same-version move from a registry package to a Git source | Diff requests review; `--fail-on source-change` can fail the audited change | +| `persisted-risk` | Same finding remains across a package version change | The finding is reported as persisted, not hidden as one resolved and one new finding | + +Run any case through the public catalog: + +```sh +make evidence CASE=license-policy +make evidence CASE=source-change-policy +make evidence CASE=persisted-risk +``` + +Each command prints the focused test command, the recorded inputs, and the +case limitation. + +## Example policy workflow + +A team wants high-severity vulnerabilities and risky source changes to block +a pull request while keeping accepted findings visible: + +```sh +bomly diff \ + --base main \ + --head HEAD \ + --enrich \ + --audit \ + --fail-on high \ + --fail-on source-change +``` + +If a package changed from a known registry source to Git or an arbitrary URL, +Bomly explains that registry-based vulnerability matching may no longer cover +it. That is a review signal, not a claim that the change is malicious. Source +formats that cannot prove origin remain unknown. + +A finding baseline does not delete matching findings. It changes their policy +status to `suppressed`, so JSON, SARIF, Guard, MCP, and human output can still +show the accepted risk. + +## Reachability cases + +The catalog includes `reachability-go`, `reachability-node`, +`reachability-python`, and `reachability-java`. + +- The Go case records the analyzer and the tier actually returned for each + vulnerability. +- The JavaScript, Python, and Java cases prove package-tier reachable and + package-tier unreachable branches. +- Every case keeps the analyzer name, tier, status, and plain-language reason + with the vulnerability. + +These cases use current advisory services, so the checked golden is a dated +observation. Re-running later may legitimately discover added, changed, or +withdrawn advisories. + +`unreachable` never means `safe`. Static analysis can miss dynamic loading, +reflection, generated code, tests, build tags, or code paths outside the +analyzer's model. Use reachability to prioritize review, not to erase the +underlying vulnerability. + +## Read-only remediation guidance + +The `remediation-read-only` case exercises the central remediation component. +For each vulnerable package, it derives: + +- whether the available evidence describes a complete fix, partial fix, no + upstream fix, or an unknown fix state; +- a recommended version only when all vulnerability evidence supports one; +- an occurrence-specific suggested action; +- package-manager advice supplied by the detector when available. + +The guidance is derived during enrichment and applies only to vulnerabilities. +License and package-policy findings stay audit results because their meaning +depends on project policy. + +Suggestions do not edit manifests, run package managers, or claim that a +change is guaranteed to work. Unknown parents and non-registry occurrences +receive manual-review guidance instead of a guessed parent upgrade. + +Reproduce the deterministic derivation matrix: + +```sh +make evidence CASE=remediation-read-only +``` + +## Limits + +- `--audit` still requires `--enrich` by default. This avoids an accidental + audit with fewer findings because matching was forgotten. +- Policy cannot make missing evidence complete. Unknown source, license, + reachability, or fix data remains unknown. +- Live advisory services can change independently. Bomly does not ship an + immutable offline advisory snapshot. +- Source changes, reachability, and remediation are different signals. None + of them alone proves that a dependency is safe or malicious. +- Bomly provides read-only remediation suggestions; it does not apply fixes. diff --git a/test/evidence/cases.json b/test/evidence/cases.json index ca2ebabc..e8594167 100644 --- a/test/evidence/cases.json +++ b/test/evidence/cases.json @@ -438,6 +438,44 @@ "This case covers one Yarn lockfile family; fallback behavior is documented separately." ] }, + { + "id": "license-policy", + "title": "Complex and invalid SPDX policy", + "area": "policy", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "fixture", + "location": "internal/auditors/license/spdx_policy_matrix_test.go", + "sha256": "f7adb4c3b59754aec61a3eabebe38854436057923bb1edf3d9497272594d91ab" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/auditors/license", + "-run", + "TestLicenseAuditorComplexSPDX|TestLicenseAuditorInvalidSPDXExpressionMatrix", + "-count=1" + ] + ], + "evidence": [ + { + "path": "internal/auditors/license/spdx_policy_matrix_test.go", + "sha256": "f7adb4c3b59754aec61a3eabebe38854436057923bb1edf3d9497272594d91ab" + } + ], + "proves": [ + "The license auditor evaluates AND, OR, nested, exception, custom-reference, and invalid SPDX expressions under allow and deny policy." + ], + "limitations": [ + "The tests prove policy evaluation after license data is present; they do not prove the accuracy of every upstream license source." + ] + }, { "id": "performance-stability", "title": "Repeated cold and warm scan measurements", @@ -472,6 +510,44 @@ "Wall time and memory are machine-specific observations and are not fixed pass-or-fail limits." ] }, + { + "id": "persisted-risk", + "title": "Risk that persists across a version change", + "area": "diff", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "fixture", + "location": "internal/engine/diff/diff_test.go", + "sha256": "8177a085ebf190011e0555f37dc89ac001856b11f0930902fb080dc8b53d5fee" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/engine/diff", + "-run", + "TestRun_SameVulnerabilityAcrossVersionBumpPersists|TestRun_SameLicenseIssueAcrossVersionBumpPersists", + "-count=1" + ] + ], + "evidence": [ + { + "path": "internal/engine/diff/diff_test.go", + "sha256": "8177a085ebf190011e0555f37dc89ac001856b11f0930902fb080dc8b53d5fee" + } + ], + "proves": [ + "A vulnerability or license finding that remains across a package version change is classified as persisted instead of one resolved and one introduced finding." + ], + "limitations": [ + "Persistence uses the canonical package-finding identity; a genuinely different advisory or rule remains a distinct finding." + ] + }, { "id": "portable-platforms", "title": "Repeated unit tests and release builds", @@ -898,6 +974,44 @@ "limitations": [ "A source change is a review signal, not proof of malicious intent, and unknown source evidence cannot be classified." ] + }, + { + "id": "vulnerability-policy", + "title": "Vulnerability policy constraints", + "area": "policy", + "evidence_level": "deterministic", + "inputs": [ + { + "kind": "fixture", + "location": "internal/auditors/vulnerability/policy_matrix_test.go", + "sha256": "79b98c8c367eebe8933328980611b6d8497c369f6be5b55e57ef25405697c7a8" + } + ], + "required_tools": [ + "go" + ], + "reproduce": [ + [ + "go", + "test", + "./internal/auditors/vulnerability", + "-run", + "TestAuditorSeverityReachabilityExploitabilityAndAllowlistMatrix", + "-count=1" + ] + ], + "evidence": [ + { + "path": "internal/auditors/vulnerability/policy_matrix_test.go", + "sha256": "79b98c8c367eebe8933328980611b6d8497c369f6be5b55e57ef25405697c7a8" + } + ], + "proves": [ + "The vulnerability auditor composes severity, reachability, known exploitation, and advisory allowlists with the documented repeated-constraint behavior." + ], + "limitations": [ + "Policy can only evaluate vulnerability, reachability, and exploitation evidence that is present in the registry." + ] } ] } From 3f15bf0a77b038df791e40ec9b3a8b134941ea9a Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 29 Jul 2026 03:43:25 -0700 Subject: [PATCH 4/6] Document target and operational assurance evidence Add the public evidence guide and explain local, Git, container, and SBOM target cases. Record dated validator and portable workflow evidence, clarify that portable assurance runs unit tests and release builds, and document repeatable cold and warm performance measurements with their limits. --- README.md | 5 +- docs/EVIDENCE.md | 91 +++++++++++++++++ docs/README.md | 1 + docs/evidence/TARGETS_AND_OPERATIONS.md | 130 ++++++++++++++++++++++++ docs/manifest.json | 6 ++ 5 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 docs/EVIDENCE.md create mode 100644 docs/evidence/TARGETS_AND_OPERATIONS.md diff --git a/README.md b/README.md index ed86d287..cdb68967 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,10 @@ Bomly reads manifests, lockfiles, package-manager output, container layers, or e | Can CI fail on high-severity findings? | `bomly scan --enrich --audit --fail-on high --format sarif` | | Can I triage reachable findings first? | `bomly scan --enrich --audit --analyze --fail-on high --fail-on reachable` | -For more recipes, see [Getting Started](docs/GETTING_STARTED.md) and [Use Cases](docs/USE_CASES.md). +For more recipes, see [Getting Started](docs/GETTING_STARTED.md) and +[Use Cases](docs/USE_CASES.md). To review the public inputs, commands, expected +results, and limitations behind important behavior claims, see +[Reproducible Evidence](docs/EVIDENCE.md). ## Explore Interactively diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md new file mode 100644 index 00000000..6ef22180 --- /dev/null +++ b/docs/EVIDENCE.md @@ -0,0 +1,91 @@ +# Reproducible evidence + +Bomly publishes the inputs, commands, expected artifacts, and limitations +behind important behavior claims. The goal is to make a claim reviewable +without private infrastructure or a one-off demonstration. + +The machine-readable catalog is +[`test/evidence/cases.json`](../test/evidence/cases.json). Check it from a +repository checkout: + +```sh +make evidence +``` + +To inspect one case: + +```sh +make evidence CASE=graph-npm +``` + +This verifies the recorded checksums and prints the exact command to reproduce +the case. + +## Evidence levels + +| Level | Meaning | +| --- | --- | +| Deterministic | Uses checked-in inputs or local services and compares a stable normalized result | +| Pinned input | Uses a public repository at a recorded commit; local tools or artifact registries can still affect build-tool-backed resolution | +| Live service | Uses a pinned project with current advisory data; the result is a dated observation | +| Manual assurance | Runs a separately started GitHub Actions workflow and saves its detailed report as an artifact | + +Each catalog case must state both what it proves and what it does not prove. +Remote Git inputs include a full commit revision. Fixtures, workflows, and +expected results include SHA-256 checksums. + +## Case studies + +- [Dependency graph evidence](evidence/DEPENDENCY_GRAPHS.md) covers npm, pnpm, + Yarn, Bun, Go, Python, and Maven graphs, plus visible degraded fallback + behavior. +- [Policy and vulnerability-guidance evidence](evidence/POLICY_AND_GUIDANCE.md) + covers vulnerability and SPDX policy, baselines, source changes, persisted + findings, reachability tiers, and read-only remediation suggestions. +- [Targets and operational assurance](evidence/TARGETS_AND_OPERATIONS.md) + covers local and Git projects, containers, SBOM ingestion and validation, + repeated unit tests across supported systems, release builds, and repeatable + performance measurements. + +## Dated workflow evidence + +The +[SBOM interoperability run from July 24, 2026](https://github.com/bomly-dev/bomly-cli/actions/runs/30057587653) +completed successfully at commit +`9530b9f3bfcb3fe1d2748fa2bcfadb5e53e3346c`. The workflow generated canonical +SPDX 2.3 and CycloneDX 1.6 documents and checked them with its recorded +checksum-pinned validators. + +The +[portable stability run from July 24, 2026](https://github.com/bomly-dev/bomly-cli/actions/runs/30065452505) +completed successfully at commit +`e6bc5235f85dc909b6bc73f6ba9eb82c22c44ac4`. It repeated Go unit tests on +Linux, macOS, and Windows, repeated the Java-related and full Linux unit-test +suites, and built every release target. It did not run remote smoke tests. + +Workflow summaries explain what ran, the result, and where to inspect failures. +Their downloadable artifacts retain detailed commands, versions, diagnostics, +and hashes where the workflow produces a run manifest. + +## How to read the results + +- A checked golden proves the normalized result for its recorded input. It is + not a promise that every project in that ecosystem has the same fidelity. +- A live enrichment golden can change when advisory services add, correct, or + withdraw records. Bomly does not claim an immutable offline advisory view. +- A package count alone is not graph proof. Review identities, versions, + relationships, scopes, sources, and occurrence paths. +- A fallback warning means useful evidence may still exist, but coverage can + be lower than the preferred detector. +- `unreachable` does not mean safe. The analyzer and tier define what was + checked. +- Remediation output is read-only guidance. It does not apply or validate a + package change. +- SBOM validation proves acceptance by the named validator versions for the + canonical fixtures, not lossless conversion for every producer or consumer. + +## Provenance + +The tracked cases use Bomly-owned fixtures, Bomly-owned example repositories, +or complete public input repositories. The descriptions and test structure +are written for Bomly's own graph, pipeline, and output contracts. diff --git a/docs/README.md b/docs/README.md index f1a4ee85..0c30e3a4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ Task-oriented walkthroughs. - [Installation](INSTALLATION.md) — install methods, `bomly` vs `bomly-lite`, checksum verification, uninstall - [Getting Started](GETTING_STARTED.md) — first scan, enrich, audit, diff - [Use Cases](USE_CASES.md) — recipes for PR gates, SBOMs, triage, license and offline scans +- [Reproducible Evidence](EVIDENCE.md) — public inputs, commands, results, and limitations behind important behavior claims - [Scan Targets](SCAN_TARGETS.md) — directories, Git repos, containers, SBOMs - [Output Formats](OUTPUT_FORMATS.md) — text, JSON, SARIF, SBOM - [SBOM Formats](SBOM.md) — SPDX vs. CycloneDX, write and ingest diff --git a/docs/evidence/TARGETS_AND_OPERATIONS.md b/docs/evidence/TARGETS_AND_OPERATIONS.md new file mode 100644 index 00000000..5f2a6fc5 --- /dev/null +++ b/docs/evidence/TARGETS_AND_OPERATIONS.md @@ -0,0 +1,130 @@ +# Targets and operational assurance + +Bomly accepts local projects, Git repositories, container images, and existing +SBOMs. The public evidence uses the same engine paths as the CLI and states +where the input itself is not immutable. + +## Target cases + +| Target | Public case | What is checked | +| --- | --- | --- | +| Local project | `baseline-policy` | A public Git fixture is materialized locally, then scanned and audited through `--path` | +| Git repository | `graph-npm`, `graph-go`, and the other graph cases | The CLI clones a recorded commit and runs the selected detector | +| Container image | `container-inventory` | Built-in inventory reads packages from the checked Alpine image | +| SPDX SBOM | `sbom-spdx-ingest` | The checked SPDX 2.3 graph is ingested through the SBOM detector | +| CycloneDX SBOM | `sbom-cyclonedx-ingest` | The checked CycloneDX 1.6 graph is ingested through the SBOM detector | + +Inspect or reproduce one: + +```sh +make evidence CASE=container-inventory +make evidence CASE=sbom-spdx-ingest +``` + +The container smoke case currently uses `alpine:3.20`. The tag can move, so +the checked-in golden is explicitly a snapshot rather than an immutable image +claim. The Git cases separately record the full commit behind their readable +tag or ref. + +## Example SBOM workflow + +A release engineer receives a supplier SBOM and wants to apply the same policy +used for source scans: + +```sh +bomly scan \ + --sbom \ + --path supplier.spdx.json \ + --enrich \ + --audit \ + --fail-on high +``` + +The deterministic ingestion cases check Bomly's internal graph. The manually +started interoperability workflow adds an external check: + +```sh +gh workflow run sbom-interoperability.yml +``` + +It generates canonical SPDX 2.3 and CycloneDX 1.6 files, verifies the +downloaded validator checksums, runs the named official validators, and saves +the generated files plus a `bomly.sbom-assurance-run/v1` report. See +[`test/assurance/SBOM_INTEROPERABILITY.md`](../../test/assurance/SBOM_INTEROPERABILITY.md) +for the workflow summary and failure-investigation steps. + +The public catalog records the workflow checksum under +`sbom-interoperability`. Validator versions and download checksums stay in the +workflow so changing either requires an intentional evidence update. + +## Supported-system checks + +The `portable-platforms` case starts: + +```sh +gh workflow run portable-assurance.yml +``` + +This workflow: + +- runs the Go unit tests twice on Linux, macOS, and Windows; +- repeats Java-related unit tests ten times; +- repeats all Go unit tests five more times on Linux; +- builds full and lightweight binaries for every release target. + +It is important to be precise: this is unit-test and build assurance. It does +not run smoke tests against public repositories, container registries, or +advisory services. The workflow summary identifies each group and explains +how to open failed logs. See +[`test/assurance/BENCHMARK_RUNS.md`](../../test/assurance/BENCHMARK_RUNS.md) +for the full description. + +## Repeatable performance measurements + +Run: + +```sh +make benchmark-samples +``` + +The `performance-stability` case uses the checked SPDX fixture and the +lightweight Bomly binary. It records five isolated cold-cache scans and five +shared-cache warm scans under `.benchmark-runs/performance`. + +The resulting `bomly.benchmark-run/v1` report includes: + +- repository and executable revisions and hashes; +- host and Go runtime details; +- exact command, working directory, cache mode, and network state; +- exit status, output size and hashes, timing, and peak memory for every run; +- median, variation, and an approximate 95% confidence interval. + +The stable gates are successful exit status, normalized output consistency, +and an optional explicit output-size cap. Wall time and memory remain +machine-specific measurements for review rather than universal limits. + +## Example release-confidence workflow + +Before a broad release: + +1. Run `make test` and the relevant pinned smoke slices. +2. Run `make benchmark-samples` and compare the report with the previous run + from a comparable host. +3. Start the portable workflow and read its Summary page. +4. If SBOM output changed, start the interoperability workflow and inspect its + generated artifact hashes and validator results. +5. Record the repository commit with every retained workflow or benchmark + report. + +## Limits + +- Remote services and package registries can be temporarily unavailable. +- Build-tool-backed graph resolution can vary with tool versions; the catalog + states required tools and checked source revisions. +- Local repository scans can contain any number of individually bounded + files. Per-file parser limits do not create a total project-size limit. +- Portable Git options bound time and checkout validation but cannot reliably + cap transfer bytes or `.git` object storage before checkout completes. +- Workflow artifacts have retention periods. The workflow file, pinned tools, + reproduction command, and result link remain public after an artifact + expires, but the raw artifact may need to be regenerated. diff --git a/docs/manifest.json b/docs/manifest.json index 196b1cdb..8f321c52 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -45,6 +45,12 @@ "description": "Recipes for PR gates, SBOMs, triage, and license and offline scans.", "group": "start" }, + { + "slug": "evidence", + "title": "Reproducible evidence", + "description": "Public inputs, commands, results, and limitations behind important behavior claims.", + "group": "start" + }, { "slug": "scan-targets", "title": "Scan targets", From 9eb418ccc1cd0c4bc217b8167e00c2babbbb79c4 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 29 Jul 2026 04:05:38 -0700 Subject: [PATCH 5/6] Address public evidence review feedback Harden evidence validation against blank claims, mutable pinned container tags, absolute catalog path mistakes, and symlink escapes. Pin manual workflow examples to the matching release definition and align evidence navigation and fallback wording with the catalog. --- docs/EVIDENCE.md | 5 +- docs/README.md | 2 +- docs/evidence/TARGETS_AND_OPERATIONS.md | 6 ++- docs/manifest.json | 3 +- internal/tools/publicevidence/main.go | 56 +++++++++++++++---- internal/tools/publicevidence/main_test.go | 63 ++++++++++++++++++++++ test/evidence/README.md | 2 + test/evidence/cases.json | 10 ++-- 8 files changed, 129 insertions(+), 18 deletions(-) diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index 6ef22180..07d28707 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -27,6 +27,7 @@ the case. | --- | --- | | Deterministic | Uses checked-in inputs or local services and compares a stable normalized result | | Pinned input | Uses a public repository at a recorded commit; local tools or artifact registries can still affect build-tool-backed resolution | +| Snapshot | Records the normalized result of an input that can move, such as a container tag | | Live service | Uses a pinned project with current advisory data; the result is a dated observation | | Manual assurance | Runs a separately started GitHub Actions workflow and saves its detailed report as an artifact | @@ -37,8 +38,8 @@ expected results include SHA-256 checksums. ## Case studies - [Dependency graph evidence](evidence/DEPENDENCY_GRAPHS.md) covers npm, pnpm, - Yarn, Bun, Go, Python, and Maven graphs, plus visible degraded fallback - behavior. + Yarn, Bun, Go, Python, and Maven graphs. A separate deterministic + `degraded-detector-fallback` case covers visible fallback orchestration. - [Policy and vulnerability-guidance evidence](evidence/POLICY_AND_GUIDANCE.md) covers vulnerability and SPDX policy, baselines, source changes, persisted findings, reachability tiers, and read-only remediation suggestions. diff --git a/docs/README.md b/docs/README.md index 0c30e3a4..ebff5047 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,6 @@ Task-oriented walkthroughs. - [Installation](INSTALLATION.md) — install methods, `bomly` vs `bomly-lite`, checksum verification, uninstall - [Getting Started](GETTING_STARTED.md) — first scan, enrich, audit, diff - [Use Cases](USE_CASES.md) — recipes for PR gates, SBOMs, triage, license and offline scans -- [Reproducible Evidence](EVIDENCE.md) — public inputs, commands, results, and limitations behind important behavior claims - [Scan Targets](SCAN_TARGETS.md) — directories, Git repos, containers, SBOMs - [Output Formats](OUTPUT_FORMATS.md) — text, JSON, SARIF, SBOM - [SBOM Formats](SBOM.md) — SPDX vs. CycloneDX, write and ingest @@ -32,6 +31,7 @@ How Bomly thinks about your project. - [Reachability](REACHABILITY.md) — narrowing findings to code your app actually calls - [MCP Server](MCP.md) — connect Bomly to Claude Code, Cursor, VS Code, or another MCP client - [Plugins](PLUGINS.md) — install, trust, configure, and package external plugins +- [Reproducible Evidence](EVIDENCE.md) — public inputs, commands, results, and limitations behind important behavior claims - Plugin implementation guides: [detector](plugins/how-to-implement-detector.md), [matcher](plugins/how-to-implement-matcher.md), [auditor](plugins/how-to-implement-auditor.md) - Example plugin repos: [Bun detector](https://github.com/bomly-dev/bomly-plugin-bun-lock-detector), [ClearlyDefined matcher](https://github.com/bomly-dev/bomly-plugin-clearlydefined-matcher), [EOL lifecycle matcher](https://github.com/bomly-dev/bomly-plugin-eol-matcher), [Meme auditor](https://github.com/bomly-dev/bomly-plugin-meme-auditor) - [Glossary](GLOSSARY.md) — every term, one sentence each diff --git a/docs/evidence/TARGETS_AND_OPERATIONS.md b/docs/evidence/TARGETS_AND_OPERATIONS.md index 5f2a6fc5..20043dc8 100644 --- a/docs/evidence/TARGETS_AND_OPERATIONS.md +++ b/docs/evidence/TARGETS_AND_OPERATIONS.md @@ -44,7 +44,7 @@ The deterministic ingestion cases check Bomly's internal graph. The manually started interoperability workflow adds an external check: ```sh -gh workflow run sbom-interoperability.yml +gh workflow run sbom-interoperability.yml --ref v0.20.0 ``` It generates canonical SPDX 2.3 and CycloneDX 1.6 files, verifies the @@ -56,13 +56,15 @@ for the workflow summary and failure-investigation steps. The public catalog records the workflow checksum under `sbom-interoperability`. Validator versions and download checksums stay in the workflow so changing either requires an intentional evidence update. +The `v0.20.0` release tag contains the recorded workflow definition, so this +command does not silently switch to a later default-branch workflow. ## Supported-system checks The `portable-platforms` case starts: ```sh -gh workflow run portable-assurance.yml +gh workflow run portable-assurance.yml --ref v0.20.0 ``` This workflow: diff --git a/docs/manifest.json b/docs/manifest.json index 8f321c52..5a17390b 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -49,7 +49,8 @@ "slug": "evidence", "title": "Reproducible evidence", "description": "Public inputs, commands, results, and limitations behind important behavior claims.", - "group": "start" + "group": "concepts", + "hasChildren": true }, { "slug": "scan-targets", diff --git a/internal/tools/publicevidence/main.go b/internal/tools/publicevidence/main.go index f5e77d3c..441cbc7d 100644 --- a/internal/tools/publicevidence/main.go +++ b/internal/tools/publicevidence/main.go @@ -23,9 +23,10 @@ const ( ) var ( - caseIDPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) - revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) - hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + caseIDPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) + revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + containerDigestPattern = regexp.MustCompile(`@sha256:[0-9a-f]{64}$`) ) type catalog struct { @@ -68,7 +69,8 @@ func main() { if err != nil { exitError(err) } - loaded, err := loadCatalog(filepath.Join(root, filepath.FromSlash(*catalogPath))) + resolvedCatalog := resolveCatalogPath(root, *catalogPath) + loaded, err := loadCatalog(resolvedCatalog) if err != nil { exitError(err) } @@ -109,6 +111,14 @@ func repositoryRoot() (string, error) { } } +func resolveCatalogPath(root, catalogPath string) string { + resolved := filepath.FromSlash(catalogPath) + if filepath.IsAbs(resolved) { + return resolved + } + return filepath.Join(root, resolved) +} + func loadCatalog(path string) (catalog, error) { file, err := os.Open(path) if err != nil { @@ -175,7 +185,7 @@ func validateCase(root string, current evidenceCase) error { return errors.New("title and area are required") } switch current.EvidenceLevel { - case "deterministic", "pinned-input", "live-service", "manual-assurance": + case "deterministic", "pinned-input", "live-service", "manual-assurance", "snapshot": default: return fmt.Errorf("unsupported evidence level %q", current.EvidenceLevel) } @@ -183,7 +193,7 @@ func validateCase(root string, current evidenceCase) error { return errors.New("at least one input is required") } for _, item := range current.Inputs { - if err := validateInput(root, item); err != nil { + if err := validateInput(root, current.EvidenceLevel, item); err != nil { return err } } @@ -211,10 +221,20 @@ func validateCase(root string, current evidenceCase) error { if len(current.Proves) == 0 || len(current.Limitations) == 0 { return errors.New("proves and limitations must both be explicit") } + for _, claim := range current.Proves { + if strings.TrimSpace(claim) == "" { + return errors.New("proves and limitations cannot contain blank entries") + } + } + for _, limitation := range current.Limitations { + if strings.TrimSpace(limitation) == "" { + return errors.New("proves and limitations cannot contain blank entries") + } + } return nil } -func validateInput(root string, current input) error { +func validateInput(root, evidenceLevel string, current input) error { if strings.TrimSpace(current.Location) == "" { return errors.New("input location is required") } @@ -232,6 +252,9 @@ func validateInput(root string, current input) error { if current.Ref == "" { return errors.New("container input requires an image reference") } + if evidenceLevel == "pinned-input" && !containerDigestPattern.MatchString(current.Ref) { + return errors.New("pinned container input requires an immutable sha256 digest") + } case "workflow": if !hashPattern.MatchString(current.SHA256) { return errors.New("workflow input requires a SHA-256 hash") @@ -251,15 +274,30 @@ func validateArtifact(root string, item artifact) error { if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { return fmt.Errorf("artifact path %q must stay inside the repository", item.Path) } + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return fmt.Errorf("resolve repository root: %w", err) + } path := filepath.Join(root, clean) - info, err := os.Stat(path) + resolvedPath, err := filepath.EvalSymlinks(path) + if err != nil { + return fmt.Errorf("resolve artifact %q: %w", item.Path, err) + } + relative, err := filepath.Rel(resolvedRoot, resolvedPath) + if err != nil { + return fmt.Errorf("resolve artifact %q relative to repository: %w", item.Path, err) + } + if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("artifact path %q resolves outside the repository", item.Path) + } + info, err := os.Stat(resolvedPath) if err != nil { return fmt.Errorf("inspect artifact %q: %w", item.Path, err) } if !info.Mode().IsRegular() { return fmt.Errorf("artifact %q is not a regular file", item.Path) } - data, err := os.ReadFile(path) + data, err := os.ReadFile(resolvedPath) if err != nil { return fmt.Errorf("read artifact %q: %w", item.Path, err) } diff --git a/internal/tools/publicevidence/main_test.go b/internal/tools/publicevidence/main_test.go index f0869479..4b76beb4 100644 --- a/internal/tools/publicevidence/main_test.go +++ b/internal/tools/publicevidence/main_test.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "os" "path/filepath" + "runtime" "strings" "testing" ) @@ -44,6 +45,16 @@ func TestLoadAndValidateCatalog(t *testing.T) { if err := validateCatalog(root, loaded); err != nil { t.Fatal(err) } + + loaded.Cases[0].Proves = []string{" "} + if err := validateCatalog(root, loaded); err == nil || !strings.Contains(err.Error(), "blank entries") { + t.Fatalf("validateCatalog() blank proof error = %v", err) + } + loaded.Cases[0].Proves = []string{"The example succeeds."} + loaded.Cases[0].Limitations = []string{"\t"} + if err := validateCatalog(root, loaded); err == nil || !strings.Contains(err.Error(), "blank entries") { + t.Fatalf("validateCatalog() blank limitation error = %v", err) + } } func TestValidateCatalogRejectsUnpinnedGitAndChangedArtifact(t *testing.T) { @@ -101,6 +112,58 @@ func TestValidateCatalogRejectsUnsortedAndUnknownFields(t *testing.T) { } } +func TestValidateInputRequiresDigestForPinnedContainer(t *testing.T) { + tagged := input{Kind: "container", Location: "Docker Hub", Ref: "alpine:3.20"} + if err := validateInput(t.TempDir(), "pinned-input", tagged); err == nil || + !strings.Contains(err.Error(), "immutable sha256 digest") { + t.Fatalf("validateInput() tagged pinned container error = %v", err) + } + if err := validateInput(t.TempDir(), "snapshot", tagged); err != nil { + t.Fatalf("validateInput() snapshot tag error = %v", err) + } + digested := tagged + digested.Ref = "alpine@sha256:" + strings.Repeat("a", 64) + if err := validateInput(t.TempDir(), "pinned-input", digested); err != nil { + t.Fatalf("validateInput() digest error = %v", err) + } +} + +func TestValidateArtifactRejectsSymlinkOutsideRepository(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink creation requires elevated privileges on Windows") + } + root := t.TempDir() + data := []byte("outside") + outside := filepath.Join(t.TempDir(), "result.json") + if err := os.WriteFile(outside, data, 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "result.json") + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + err := validateArtifact(root, artifact{ + Path: "result.json", + SHA256: hex.EncodeToString(sum[:]), + }) + if err == nil || !strings.Contains(err.Error(), "resolves outside the repository") { + t.Fatalf("validateArtifact() error = %v", err) + } +} + +func TestResolveCatalogPathHonorsAbsolutePath(t *testing.T) { + root := t.TempDir() + absolute := filepath.Join(t.TempDir(), "cases.json") + if got := resolveCatalogPath(root, absolute); got != absolute { + t.Fatalf("resolveCatalogPath() absolute = %q, want %q", got, absolute) + } + wantRelative := filepath.Join(root, "test", "evidence", "cases.json") + if got := resolveCatalogPath(root, "test/evidence/cases.json"); got != wantRelative { + t.Fatalf("resolveCatalogPath() relative = %q, want %q", got, wantRelative) + } +} + func TestShellCommandQuotesPatterns(t *testing.T) { got := shellCommand([]string{"go", "test", "-run", "TestScan$/scan-npm$"}) want := "go test -run 'TestScan$/scan-npm$'" diff --git a/test/evidence/README.md b/test/evidence/README.md index 2fca08af..1c642b0a 100644 --- a/test/evidence/README.md +++ b/test/evidence/README.md @@ -30,6 +30,8 @@ The checker verifies that: the same normalized result. - `pinned-input` uses a public repository revision. Package-manager tools or registries can still affect build-tool-backed resolution. +- `snapshot` records the normalized result of an input that can move, such as + a container tag. - `live-service` combines a pinned project with current advisory data. It is a dated observation because advisory services change. - `manual-assurance` starts a GitHub Actions workflow and stores the detailed diff --git a/test/evidence/cases.json b/test/evidence/cases.json index e8594167..093e191c 100644 --- a/test/evidence/cases.json +++ b/test/evidence/cases.json @@ -50,7 +50,7 @@ "id": "container-inventory", "title": "Container package inventory", "area": "targets", - "evidence_level": "pinned-input", + "evidence_level": "snapshot", "inputs": [ { "kind": "container", @@ -568,7 +568,9 @@ "gh", "workflow", "run", - "portable-assurance.yml" + "portable-assurance.yml", + "--ref", + "v0.20.0" ] ], "evidence": [ @@ -865,7 +867,9 @@ "gh", "workflow", "run", - "sbom-interoperability.yml" + "sbom-interoperability.yml", + "--ref", + "v0.20.0" ] ], "evidence": [ From 5f8db68665612c4c4dedecc784a14441119509fa Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Wed, 29 Jul 2026 17:21:43 -0700 Subject: [PATCH 6/6] Move reproducible evidence under reference docs Keep advanced assurance material out of the everyday concepts section while retaining child navigation for the detailed evidence pages. --- docs/README.md | 2 +- docs/manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index ebff5047..af964ced 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,7 +31,6 @@ How Bomly thinks about your project. - [Reachability](REACHABILITY.md) — narrowing findings to code your app actually calls - [MCP Server](MCP.md) — connect Bomly to Claude Code, Cursor, VS Code, or another MCP client - [Plugins](PLUGINS.md) — install, trust, configure, and package external plugins -- [Reproducible Evidence](EVIDENCE.md) — public inputs, commands, results, and limitations behind important behavior claims - Plugin implementation guides: [detector](plugins/how-to-implement-detector.md), [matcher](plugins/how-to-implement-matcher.md), [auditor](plugins/how-to-implement-auditor.md) - Example plugin repos: [Bun detector](https://github.com/bomly-dev/bomly-plugin-bun-lock-detector), [ClearlyDefined matcher](https://github.com/bomly-dev/bomly-plugin-clearlydefined-matcher), [EOL lifecycle matcher](https://github.com/bomly-dev/bomly-plugin-eol-matcher), [Meme auditor](https://github.com/bomly-dev/bomly-plugin-meme-auditor) - [Glossary](GLOSSARY.md) — every term, one sentence each @@ -47,6 +46,7 @@ Generated from code. Treat as authoritative. - [Matcher Reference](matchers/) — per-matcher behavior, cache, output - [Auditor Reference](auditors/) — per-auditor options, examples, limitations - [JSON Schemas](SCHEMAS.md) — scan, explain, diff output shapes +- [Reproducible Evidence](EVIDENCE.md) — public inputs, commands, results, and limitations behind important behavior claims ## Project diff --git a/docs/manifest.json b/docs/manifest.json index 5a17390b..19a13410 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -49,7 +49,7 @@ "slug": "evidence", "title": "Reproducible evidence", "description": "Public inputs, commands, results, and limitations behind important behavior claims.", - "group": "concepts", + "group": "reference", "hasChildren": true }, {