From dfeb33ae6656a7b3bf4e49aa4bec0b572da89f51 Mon Sep 17 00:00:00 2001 From: Abhineshhh Date: Fri, 17 Jul 2026 01:25:49 +0530 Subject: [PATCH] fix: persist comprehensive Trivy secrets and misconfigs Comprehensive scans counted secrets/misconfigurations but never built SecurityFindings, so the security_findings table stayed empty and detail reports could not surface them. Map Trivy secrets and misconfigs into domain findings, store them on the image record, load them (and capabilities) in QueryAllImageDetails, and emit them in the detailed JSON report. Fixes #73 --- docs/detailed-report.md | 11 ++- pkg/domain/models.go | 5 +- pkg/infrastructure/database/repository.go | 67 ++++++++++++++++- .../database/repository_test.go | 71 +++++++++++++++++++ pkg/infrastructure/report/json_detail.go | 47 ++++++++++-- pkg/infrastructure/report/json_detail_test.go | 20 +++++- pkg/infrastructure/scanner/analyzer.go | 61 ++++++++-------- pkg/infrastructure/scanner/trivy.go | 41 ++++++++--- pkg/infrastructure/scanner/trivy_test.go | 16 +++-- 9 files changed, 279 insertions(+), 60 deletions(-) diff --git a/docs/detailed-report.md b/docs/detailed-report.md index d6ed512..58f547f 100644 --- a/docs/detailed-report.md +++ b/docs/detailed-report.md @@ -52,15 +52,20 @@ The detailed report is written alongside the other reports as `docs/daily_recomm ], "packageManagers": [ { "name": "pip", "version": "24.0", "language": "python" } - ] + ], + "securityFindings": [ + { "findingType": "secret", "severity": "HIGH", "ruleId": "generic-api-key", + "title": "API key", "category": "token" } + ], + "capabilities": ["ssl", "http_client"] } ] } ``` -> **Optional fields:** Some fields are omitted when empty or unknown, including `registry`, `repository`, `tag`, `digest`, `scanTimestamp`, `baseOS.version`, `languages[].majorMinor`, `languages[].packageName`, `languages[].packageType`, `vulnerabilities[].fixedVersion`, and `vulnerabilities[].description`. Array fields (`languages`, `vulnerabilities`, `systemPackages`, `packageManagers`) are always present as empty arrays `[]`, never null. +> **Optional fields:** Some fields are omitted when empty or unknown, including `registry`, `repository`, `tag`, `digest`, `scanTimestamp`, `baseOS.version`, `languages[].majorMinor`, `languages[].packageName`, `languages[].packageType`, `vulnerabilities[].fixedVersion`, `vulnerabilities[].description`, and most `securityFindings[]` fields when empty. Array fields (`languages`, `vulnerabilities`, `systemPackages`, `packageManagers`, `securityFindings`, `capabilities`) are always present as empty arrays `[]`, never null. > -> **Scope:** The current detailed report includes system packages, package managers, detected languages, and CVE vulnerabilities. Comprehensive scan findings (secrets, misconfigurations) and capabilities are not included. +> **Scope:** The detailed report includes system packages, package managers, detected languages, CVE vulnerabilities, capabilities, and comprehensive scan findings (secrets and misconfigurations when present in the database). ## Querying with jq diff --git a/pkg/domain/models.go b/pkg/domain/models.go index 0a51bf2..aedd6ea 100644 --- a/pkg/domain/models.go +++ b/pkg/domain/models.go @@ -215,8 +215,9 @@ type TrivyResult struct { SecretsFound int ConfigIssues int LicenseIssues int - BaseOSFamily string // e.g. "azurelinux", "ubuntu", "debian" - BaseOSVersion string // e.g. "3.0", "22.04", "12.13" + SecurityFindings []SecurityFinding // secrets + misconfigurations + BaseOSFamily string // e.g. "azurelinux", "ubuntu", "debian" + BaseOSVersion string // e.g. "3.0", "22.04", "12.13" } // RecommendedImage holds a ranked image for the report output. diff --git a/pkg/infrastructure/database/repository.go b/pkg/infrastructure/database/repository.go index ef22560..e0ce58d 100644 --- a/pkg/infrastructure/database/repository.go +++ b/pkg/infrastructure/database/repository.go @@ -361,7 +361,8 @@ func (r *Repository) ClearDatabase() error { } // QueryAllImageDetails returns every image with its child data populated -// (languages, vulnerabilities, system packages, package managers). +// (languages, vulnerabilities, system packages, package managers, +// security findings, capabilities). // Uses batched eager loading within a read transaction for snapshot consistency. func (r *Repository) QueryAllImageDetails() ([]domain.ImageRecord, error) { tx, err := r.db.Begin() @@ -550,6 +551,70 @@ func (r *Repository) QueryAllImageDetails() ([]domain.ImageRecord, error) { return nil, fmt.Errorf("iterating package managers: %w", err) } + // Step 6: Load security findings (secrets / misconfigurations) + sfRows, err := tx.Query(` + SELECT image_id, COALESCE(finding_type, ''), COALESCE(severity, ''), + COALESCE(rule_id, ''), COALESCE(title, ''), COALESCE(description, ''), + COALESCE(file_path, ''), COALESCE(category, ''), COALESCE(message, '') + FROM security_findings ORDER BY image_id, id`) + if err != nil { + return nil, fmt.Errorf("querying security findings: %w", err) + } + + for sfRows.Next() { + var imageID int64 + var sf domain.SecurityFinding + + if err := sfRows.Scan(&imageID, &sf.FindingType, &sf.Severity, + &sf.RuleID, &sf.Title, &sf.Description, &sf.FilePath, + &sf.Category, &sf.Message); err != nil { + _ = sfRows.Close() + return nil, fmt.Errorf("scanning security finding: %w", err) + } + + if img, ok := imageMap[imageID]; ok { + img.SecurityFindings = append(img.SecurityFindings, sf) + } + } + + if err := sfRows.Close(); err != nil { + return nil, fmt.Errorf("closing security finding rows: %w", err) + } + + if err := sfRows.Err(); err != nil { + return nil, fmt.Errorf("iterating security findings: %w", err) + } + + // Step 7: Load capabilities + capRows, err := tx.Query(` + SELECT image_id, COALESCE(capability, '') + FROM capabilities ORDER BY image_id, id`) + if err != nil { + return nil, fmt.Errorf("querying capabilities: %w", err) + } + + for capRows.Next() { + var imageID int64 + var c domain.Capability + + if err := capRows.Scan(&imageID, &c.Capability); err != nil { + _ = capRows.Close() + return nil, fmt.Errorf("scanning capability: %w", err) + } + + if img, ok := imageMap[imageID]; ok { + img.Capabilities = append(img.Capabilities, c) + } + } + + if err := capRows.Close(); err != nil { + return nil, fmt.Errorf("closing capability rows: %w", err) + } + + if err := capRows.Err(); err != nil { + return nil, fmt.Errorf("iterating capabilities: %w", err) + } + // Assemble in original order results := make([]domain.ImageRecord, 0, len(imageOrder)) for _, id := range imageOrder { diff --git a/pkg/infrastructure/database/repository_test.go b/pkg/infrastructure/database/repository_test.go index 7afa79d..26ec9ae 100644 --- a/pkg/infrastructure/database/repository_test.go +++ b/pkg/infrastructure/database/repository_test.go @@ -397,6 +397,77 @@ func TestQueryLanguages_BaseSortsLast(t *testing.T) { assert.Equal(t, "base", languages[2]) } +func TestInsertAndQuerySecurityFindingsAndCapabilities(t *testing.T) { + db, repo := setupTestDB(t) + defer func() { _ = db.Close() }() + + img := &domain.ImageRecord{ + Name: "mcr.microsoft.com/example:1.0", + Registry: "mcr.microsoft.com", + Repository: "example", + Tag: "1.0", + SecretsFound: 1, + ConfigIssues: 1, + Languages: []domain.Language{ + {Language: "python", Version: "3.12"}, + }, + Capabilities: []domain.Capability{ + {Capability: "ssl"}, + {Capability: "http_client"}, + }, + SecurityFindings: []domain.SecurityFinding{ + { + FindingType: "secret", + Severity: "CRITICAL", + RuleID: "aws-secret-key", + Title: "AWS secret key", + Description: "AKIA...", + Category: "AWS", + }, + { + FindingType: "misconfiguration", + Severity: "HIGH", + RuleID: "DS001", + Title: "root user", + Message: "Dockerfile runs as root", + }, + }, + } + require.NoError(t, repo.InsertImage(img)) + + details, err := repo.QueryAllImageDetails() + require.NoError(t, err) + require.Len(t, details, 1) + + got := details[0] + require.Len(t, got.SecurityFindings, 2) + assert.Equal(t, "secret", got.SecurityFindings[0].FindingType) + assert.Equal(t, "aws-secret-key", got.SecurityFindings[0].RuleID) + assert.Equal(t, "misconfiguration", got.SecurityFindings[1].FindingType) + assert.Equal(t, "DS001", got.SecurityFindings[1].RuleID) + + require.Len(t, got.Capabilities, 2) + assert.Equal(t, "ssl", got.Capabilities[0].Capability) + assert.Equal(t, "http_client", got.Capabilities[1].Capability) + + // Upsert clears and rewrites related rows + img.SecurityFindings = []domain.SecurityFinding{ + {FindingType: "secret", Severity: "LOW", RuleID: "generic-api-key", Title: "API key"}, + } + img.Capabilities = []domain.Capability{{Capability: "compression"}} + img.SecretsFound = 1 + img.ConfigIssues = 0 + require.NoError(t, repo.InsertImage(img)) + + details, err = repo.QueryAllImageDetails() + require.NoError(t, err) + require.Len(t, details, 1) + require.Len(t, details[0].SecurityFindings, 1) + assert.Equal(t, "generic-api-key", details[0].SecurityFindings[0].RuleID) + require.Len(t, details[0].Capabilities, 1) + assert.Equal(t, "compression", details[0].Capabilities[0].Capability) +} + // TestInsertImage_RollbackOnClearFailure ensures a failure while clearing // related tables rolls back the whole upsert and releases the connection. // diff --git a/pkg/infrastructure/report/json_detail.go b/pkg/infrastructure/report/json_detail.go index 3ef0fed..6c89feb 100644 --- a/pkg/infrastructure/report/json_detail.go +++ b/pkg/infrastructure/report/json_detail.go @@ -59,6 +59,8 @@ type DetailImageEntry struct { Vulnerabilities []DetailVulnEntry `json:"vulnerabilities"` SystemPackages []DetailSystemPkgEntry `json:"systemPackages"` PackageManagers []DetailPkgManagerEntry `json:"packageManagers"` + SecurityFindings []DetailSecurityFinding `json:"securityFindings"` + Capabilities []string `json:"capabilities"` } // DetailBaseOS holds the OS information for an image. @@ -113,6 +115,18 @@ type DetailPkgManagerEntry struct { Language string `json:"language,omitempty"` } +// DetailSecurityFinding represents a secret or misconfiguration from comprehensive scans. +type DetailSecurityFinding struct { + FindingType string `json:"findingType"` + Severity string `json:"severity,omitempty"` + RuleID string `json:"ruleId,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + FilePath string `json:"filePath,omitempty"` + Category string `json:"category,omitempty"` + Message string `json:"message,omitempty"` +} + // GenerateDetailJSONReport produces a detailed JSON report with per-image // package inventories, vulnerability breakdowns, and detected languages. func GenerateDetailJSONReport(repo *database.Repository, outputPath string) error { @@ -131,8 +145,8 @@ func GenerateDetailJSONReport(repo *database.Repository, outputPath string) erro report := DetailJSONReport{ SchemaVersion: 1, GeneratedAt: time.Now().UTC().Format(time.RFC3339), - ImageCount: len(images), - Images: make([]DetailImageEntry, 0, len(images)), + ImageCount: len(images), + Images: make([]DetailImageEntry, 0, len(images)), } for _, img := range images { @@ -160,10 +174,12 @@ func GenerateDetailJSONReport(repo *database.Repository, outputPath string) erro Negligible: img.NegligibleVulnerabilities, Unknown: img.UnknownVulnerabilities, }, - Languages: make([]DetailLanguageEntry, 0, len(img.Languages)), - Vulnerabilities: make([]DetailVulnEntry, 0, len(img.Vulnerabilities)), - SystemPackages: make([]DetailSystemPkgEntry, 0, len(img.SystemPackages)), - PackageManagers: make([]DetailPkgManagerEntry, 0, len(img.PackageManagers)), + Languages: make([]DetailLanguageEntry, 0, len(img.Languages)), + Vulnerabilities: make([]DetailVulnEntry, 0, len(img.Vulnerabilities)), + SystemPackages: make([]DetailSystemPkgEntry, 0, len(img.SystemPackages)), + PackageManagers: make([]DetailPkgManagerEntry, 0, len(img.PackageManagers)), + SecurityFindings: make([]DetailSecurityFinding, 0, len(img.SecurityFindings)), + Capabilities: make([]string, 0, len(img.Capabilities)), } for _, l := range img.Languages { @@ -205,6 +221,25 @@ func GenerateDetailJSONReport(repo *database.Repository, outputPath string) erro }) } + for _, sf := range img.SecurityFindings { + entry.SecurityFindings = append(entry.SecurityFindings, DetailSecurityFinding{ + FindingType: sf.FindingType, + Severity: sf.Severity, + RuleID: sf.RuleID, + Title: sf.Title, + Description: sf.Description, + FilePath: sf.FilePath, + Category: sf.Category, + Message: sf.Message, + }) + } + + for _, c := range img.Capabilities { + if c.Capability != "" { + entry.Capabilities = append(entry.Capabilities, c.Capability) + } + } + report.Images = append(report.Images, entry) } diff --git a/pkg/infrastructure/report/json_detail_test.go b/pkg/infrastructure/report/json_detail_test.go index 561ed8b..9e5f77b 100644 --- a/pkg/infrastructure/report/json_detail_test.go +++ b/pkg/infrastructure/report/json_detail_test.go @@ -42,7 +42,7 @@ func TestGenerateDetailJSONReport_FullImage(t *testing.T) { Repository: "azurelinux/base/python", Tag: "3.12", BaseOSName: "azurelinux", BaseOSVersion: "3.0", Digest: "sha256:abc123", SizeBytes: 85000000, Layers: 5, - CreatedDate: "2025-04-15T08:30:00Z", + CreatedDate: "2025-04-15T08:30:00Z", TotalVulnerabilities: 3, CriticalVulnerabilities: 1, HighVulnerabilities: 1, MediumVulnerabilities: 1, Languages: []domain.Language{ @@ -60,6 +60,12 @@ func TestGenerateDetailJSONReport_FullImage(t *testing.T) { PackageManagers: []domain.PackageManager{ {Name: "pip", Version: "24.0", Language: "python"}, }, + Capabilities: []domain.Capability{ + {Capability: "ssl"}, + }, + SecurityFindings: []domain.SecurityFinding{ + {FindingType: "secret", Severity: "HIGH", RuleID: "generic-api-key", Title: "API key", Category: "token"}, + }, } require.NoError(t, repo.InsertImage(img)) @@ -111,6 +117,14 @@ func TestGenerateDetailJSONReport_FullImage(t *testing.T) { require.Len(t, entry.PackageManagers, 1) assert.Equal(t, "pip", entry.PackageManagers[0].Name) + + require.Len(t, entry.SecurityFindings, 1) + assert.Equal(t, "secret", entry.SecurityFindings[0].FindingType) + assert.Equal(t, "generic-api-key", entry.SecurityFindings[0].RuleID) + assert.Equal(t, "HIGH", entry.SecurityFindings[0].Severity) + + require.Len(t, entry.Capabilities, 1) + assert.Equal(t, "ssl", entry.Capabilities[0]) } func TestGenerateDetailJSONReport_EmptyDB(t *testing.T) { @@ -155,7 +169,7 @@ func TestGenerateDetailJSONReport_MultipleImages(t *testing.T) { { Name: "go-img:1.21", Registry: "r", Repository: "repo2", Tag: "1.21", BaseOSName: "azurelinux", TotalVulnerabilities: 0, - Languages: []domain.Language{{Language: "go", Version: "1.21.0"}}, + Languages: []domain.Language{{Language: "go", Version: "1.21.0"}}, }, } @@ -197,7 +211,7 @@ func TestGenerateDetailJSONReport_VulnSummaryCounts(t *testing.T) { TotalVulnerabilities: 6, CriticalVulnerabilities: 1, HighVulnerabilities: 2, MediumVulnerabilities: 1, LowVulnerabilities: 1, NegligibleVulnerabilities: 0, UnknownVulnerabilities: 1, - Languages: []domain.Language{{Language: "python", Version: "3.12"}}, + Languages: []domain.Language{{Language: "python", Version: "3.12"}}, } require.NoError(t, repo.InsertImage(img)) diff --git a/pkg/infrastructure/scanner/analyzer.go b/pkg/infrastructure/scanner/analyzer.go index b027681..71a5890 100644 --- a/pkg/infrastructure/scanner/analyzer.go +++ b/pkg/infrastructure/scanner/analyzer.go @@ -58,9 +58,9 @@ var versionPatterns = map[string]*regexp.Regexp{ // ImageAnalyzer orchestrates the full analysis of a container image. type ImageAnalyzer struct { - docker *DockerClient - comprehensive bool - cleanupImages bool + docker *DockerClient + comprehensive bool + cleanupImages bool } // NewImageAnalyzer creates a new ImageAnalyzer. @@ -136,33 +136,34 @@ func (a *ImageAnalyzer) Analyze(imageName string) (*domain.ImageAnalysis, error) TrivyResult: *trivyResult, RuntimeVersions: runtimeVersions, Image: domain.ImageRecord{ - Name: imageName, - Registry: registry, - Repository: repository, - Tag: tag, - Digest: inspectResult.Digest, - SizeBytes: imageSize, - Layers: inspectResult.Layers, - CreatedDate: inspectResult.Created, - BaseOSName: trivyResult.BaseOSFamily, - BaseOSVersion: trivyResult.BaseOSVersion, - TotalVulnerabilities: trivyResult.TotalVulnerabilities, - CriticalVulnerabilities: trivyResult.CriticalVulnerabilities, - HighVulnerabilities: trivyResult.HighVulnerabilities, - MediumVulnerabilities: trivyResult.MediumVulnerabilities, - LowVulnerabilities: trivyResult.LowVulnerabilities, - NegligibleVulnerabilities: trivyResult.NegligibleVulnerabilities, - UnknownVulnerabilities: trivyResult.UnknownVulnerabilities, - VulnerabilityScanTimestamp: time.Now().UTC().Format(time.RFC3339), - VulnerabilityScanner: "trivy", - SecretsFound: trivyResult.SecretsFound, - ConfigIssues: trivyResult.ConfigIssues, - LicenseIssues: trivyResult.LicenseIssues, - Languages: languages, - Vulnerabilities: trivyResult.Vulnerabilities, - PackageManagers: syftResult.PackageManagers, - Capabilities: syftResult.Capabilities, - SystemPackages: syftResult.SystemPackages, + Name: imageName, + Registry: registry, + Repository: repository, + Tag: tag, + Digest: inspectResult.Digest, + SizeBytes: imageSize, + Layers: inspectResult.Layers, + CreatedDate: inspectResult.Created, + BaseOSName: trivyResult.BaseOSFamily, + BaseOSVersion: trivyResult.BaseOSVersion, + TotalVulnerabilities: trivyResult.TotalVulnerabilities, + CriticalVulnerabilities: trivyResult.CriticalVulnerabilities, + HighVulnerabilities: trivyResult.HighVulnerabilities, + MediumVulnerabilities: trivyResult.MediumVulnerabilities, + LowVulnerabilities: trivyResult.LowVulnerabilities, + NegligibleVulnerabilities: trivyResult.NegligibleVulnerabilities, + UnknownVulnerabilities: trivyResult.UnknownVulnerabilities, + VulnerabilityScanTimestamp: time.Now().UTC().Format(time.RFC3339), + VulnerabilityScanner: "trivy", + SecretsFound: trivyResult.SecretsFound, + ConfigIssues: trivyResult.ConfigIssues, + LicenseIssues: trivyResult.LicenseIssues, + Languages: languages, + Vulnerabilities: trivyResult.Vulnerabilities, + PackageManagers: syftResult.PackageManagers, + Capabilities: syftResult.Capabilities, + SystemPackages: syftResult.SystemPackages, + SecurityFindings: trivyResult.SecurityFindings, }, } diff --git a/pkg/infrastructure/scanner/trivy.go b/pkg/infrastructure/scanner/trivy.go index 689d9f4..3b993af 100644 --- a/pkg/infrastructure/scanner/trivy.go +++ b/pkg/infrastructure/scanner/trivy.go @@ -49,13 +49,13 @@ type trivyOS struct { } type trivyResult struct { - Target string `json:"Target"` - Class string `json:"Class"` - Type string `json:"Type"` - Vulnerabilities []trivyVuln `json:"Vulnerabilities"` - Secrets []trivySecret `json:"Secrets"` - Misconfigs []trivyMisconfig `json:"Misconfigurations"` - Licenses []trivyLicense `json:"Licenses"` + Target string `json:"Target"` + Class string `json:"Class"` + Type string `json:"Type"` + Vulnerabilities []trivyVuln `json:"Vulnerabilities"` + Secrets []trivySecret `json:"Secrets"` + Misconfigs []trivyMisconfig `json:"Misconfigurations"` + Licenses []trivyLicense `json:"Licenses"` } type trivyCVSS struct { @@ -170,13 +170,32 @@ func parseTrivyResult(output *trivyOutput) *domain.TrivyResult { }) } - // Process secrets - result.SecretsFound += len(r.Secrets) + // Process secrets (counts + detailed findings for persistence) + for _, s := range r.Secrets { + result.SecretsFound++ + result.SecurityFindings = append(result.SecurityFindings, domain.SecurityFinding{ + FindingType: "secret", + Severity: s.Severity, + RuleID: s.RuleID, + Title: s.Title, + Description: truncateString(s.Match, 500), + Category: s.Category, + }) + } // Process misconfigurations - result.ConfigIssues += len(r.Misconfigs) + for _, m := range r.Misconfigs { + result.ConfigIssues++ + result.SecurityFindings = append(result.SecurityFindings, domain.SecurityFinding{ + FindingType: "misconfiguration", + Severity: m.Severity, + RuleID: m.ID, + Title: m.Title, + Message: m.Message, + }) + } - // Process licenses + // Process licenses (counts only — not stored as security_findings rows) result.LicenseIssues += len(r.Licenses) } diff --git a/pkg/infrastructure/scanner/trivy_test.go b/pkg/infrastructure/scanner/trivy_test.go index 478e4e4..b133ecf 100644 --- a/pkg/infrastructure/scanner/trivy_test.go +++ b/pkg/infrastructure/scanner/trivy_test.go @@ -365,13 +365,13 @@ func TestParseTrivyResult_VulnerabilityDetails(t *testing.T) { // Find the CRITICAL vuln var critVuln *struct { id, severity, pkg, pkgVer, fixedVer, desc string - cvss float64 + cvss float64 } for _, v := range result.Vulnerabilities { if v.Severity == "CRITICAL" { critVuln = &struct { id, severity, pkg, pkgVer, fixedVer, desc string - cvss float64 + cvss float64 }{v.VulnerabilityID, v.Severity, v.PackageName, v.PackageVersion, v.FixedVersion, v.Description, v.CVSSScore} break } @@ -478,6 +478,16 @@ func TestParseTrivyResult_SecretsAndMisconfigs(t *testing.T) { assert.Equal(t, 1, result.ConfigIssues) assert.Equal(t, 2, result.LicenseIssues) assert.Equal(t, 0, result.TotalVulnerabilities, "secrets/misconfigs/licenses don't count as vulns") + + require.Len(t, result.SecurityFindings, 3) + assert.Equal(t, "secret", result.SecurityFindings[0].FindingType) + assert.Equal(t, "aws-secret-key", result.SecurityFindings[0].RuleID) + assert.Equal(t, "CRITICAL", result.SecurityFindings[0].Severity) + assert.Equal(t, "secret", result.SecurityFindings[1].FindingType) + assert.Equal(t, "generic-api-key", result.SecurityFindings[1].RuleID) + assert.Equal(t, "misconfiguration", result.SecurityFindings[2].FindingType) + assert.Equal(t, "DS001", result.SecurityFindings[2].RuleID) + assert.Equal(t, "root user", result.SecurityFindings[2].Title) } // ============================================================================ @@ -604,5 +614,3 @@ func TestParseTrivyResult_OSMetadata_AllOSFamilies(t *testing.T) { }) } } - -