diff --git a/updater/fetchers/oracle/oracle.go b/updater/fetchers/oracle/oracle.go index 57923bf..2f3182d 100644 --- a/updater/fetchers/oracle/oracle.go +++ b/updater/fetchers/oracle/oracle.go @@ -1,12 +1,14 @@ package oracle import ( - "bufio" + "bytes" + "compress/bzip2" "encoding/xml" "fmt" "io" "net/http" "regexp" + "sort" "strconv" "strings" "time" @@ -20,7 +22,7 @@ import ( const ( firstConsideredELSA = 7 - ovalURI = "https://linux.oracle.com/oval/" + ovalURI = "https://linux.oracle.com/security/oval/" retryTimes = 5 ) @@ -30,7 +32,7 @@ var ( ".ksplice1.", } - elsaRegexp = regexp.MustCompile(`com.oracle.elsa-(\d+).xml`) + elsaRegexp = regexp.MustCompile(`href="(com\.oracle\.elsa-(?:all|ol(?:6|7|8|9|10))\.xml\.bz2)"`) ) type oval struct { @@ -89,6 +91,9 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) log.Info("fetching Oracle vulnerabilities") req, err := http.NewRequest("GET", ovalURI, nil) + if err != nil { + return resp, err + } req.Header.Add("User-Agent", "dbgen") client := http.Client{} r, err := client.Do(req) @@ -96,37 +101,37 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) log.Errorf("could not download Oracle's directory: %s", err) return resp, common.ErrCouldNotDownload } + defer r.Body.Close() - // Get the list of ELSAs that we have to process. - var elsaList []int - scanner := bufio.NewScanner(r.Body) - for scanner.Scan() { - line := scanner.Text() - r := elsaRegexp.FindStringSubmatch(line) - if len(r) == 2 && len(r[1]) > 4 { - year, _ := strconv.Atoi(r[1][:4]) - if year >= common.FirstYear { - elsaNo, _ := strconv.Atoi(r[1]) - elsaList = append(elsaList, elsaNo) - } - } + indexBody, err := io.ReadAll(r.Body) + if err != nil { + log.Errorf("could not read Oracle's directory: %s", err) + return resp, common.ErrCouldNotDownload + } + + feedFiles := listFeedFiles(indexBody) + if len(feedFiles) == 0 { + log.Error("Failed to find Oracle oval feeds") + return resp, fmt.Errorf("Failed to find Oracle oval feeds") } - r.Body.Close() + vulnMap := make(map[string]common.Vulnerability) - for i, elsa := range elsaList { + for i, elsaFile := range feedFiles { var vs []common.Vulnerability retry := 0 for retry <= retryTimes { - elsaFile := fmt.Sprintf("com.oracle.elsa-%d.xml", elsa) rurl := fmt.Sprintf("%s%s", ovalURI, elsaFile) client := http.Client{} - req, _ := http.NewRequest("GET", rurl, nil) + req, err := http.NewRequest("GET", rurl, nil) + if err != nil { + return resp, err + } req.Header.Add("User-Agent", "dbgen") if r, err := client.Do(req); err == nil { - vs, err = parseELSA(elsaFile, r.Body) + vs, err = parseBZ2ELSA(elsaFile, r.Body) r.Body.Close() if err == nil { @@ -148,7 +153,7 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) // Collect vulnerabilities. for _, v := range vs { - resp.Vulnerabilities = append(resp.Vulnerabilities, v) + mergeVulnerability(vulnMap, v) } // Pause to prevent the website from blacklisting us. @@ -157,6 +162,10 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) } } + for _, v := range vulnMap { + resp.Vulnerabilities = append(resp.Vulnerabilities, v) + } + if len(resp.Vulnerabilities) == 0 { log.Error("Failed to fetch Oracle oval feed") return resp, fmt.Errorf("Failed to fetch Oracle oval feed") @@ -166,11 +175,30 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) return resp, nil } +func parseBZ2ELSA(elsa string, compressedReader io.Reader) ([]common.Vulnerability, error) { + return parseELSA(elsa, bzip2.NewReader(compressedReader)) +} + func parseELSA(elsa string, ovalReader io.Reader) (vulnerabilities []common.Vulnerability, err error) { + body, err := io.ReadAll(ovalReader) + if err != nil { + return nil, err + } + + trimmed := bytes.TrimSpace(body) + if bytes.HasPrefix(trimmed, []byte(" + + +
+ +` + + vuls, err := parseELSA("com.oracle.elsa-20269999.xml", strings.NewReader(html)) + if err != nil { + t.Fatalf("expected HTML response to be skipped without error, got: %v", err) + } + if len(vuls) != 0 { + t.Fatalf("expected no vulnerabilities for skipped HTML response, got: %d", len(vuls)) + } +} + +func TestListFeedFiles(t *testing.T) { + indexHTML := ` +com.oracle.elsa-all...> +com.oracle.elsa-ol6...> +com.oracle.elsa-ol7...> +com.oracle.elsa-ol8...> +com.oracle.elsa-ol9...> +com.oracle.elsa-ol10...> +com.oracle.elsa-2026...>` + + files := listFeedFiles([]byte(indexHTML)) + expected := []string{ + "com.oracle.elsa-all.xml.bz2", + "com.oracle.elsa-ol10.xml.bz2", + "com.oracle.elsa-ol6.xml.bz2", + "com.oracle.elsa-ol7.xml.bz2", + "com.oracle.elsa-ol8.xml.bz2", + "com.oracle.elsa-ol9.xml.bz2", + } + + if len(files) != len(expected) { + t.Fatalf("expected %d files, got %d: %v", len(expected), len(files), files) + } + for i := range expected { + if files[i] != expected[i] { + t.Fatalf("expected file %d to be %q, got %q", i, expected[i], files[i]) + } + } +} + +func TestMergeVulnerabilityDeduplicatesFixedInAndCVEs(t *testing.T) { + vulnMap := make(map[string]common.Vulnerability) + version, err := common.NewVersion("1.1.1k-1") + if err != nil { + t.Fatalf("failed to parse test version: %v", err) + } + fixedIn := common.FeatureVersion{ + Feature: common.Feature{ + Name: "openssl", + Namespace: "oracle:9", + }, + Version: version, + } + v := common.Vulnerability{ + Name: "ELSA-2026-0001", + FixedIn: []common.FeatureVersion{fixedIn}, + CVEs: []common.CVE{{Name: "CVE-2026-0001"}}, + } + + mergeVulnerability(vulnMap, v) + mergeVulnerability(vulnMap, v) + + got := vulnMap[v.Name] + if len(got.FixedIn) != 1 { + t.Fatalf("expected one fixed-in entry after dedupe, got %d", len(got.FixedIn)) + } + if len(got.CVEs) != 1 { + t.Fatalf("expected one CVE after dedupe, got %d", len(got.CVEs)) + } +} diff --git a/updater/updater.go b/updater/updater.go index 040e8f9..aab8bd1 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -3,6 +3,7 @@ package updater import ( "fmt" "strings" + "time" log "github.com/sirupsen/logrus" @@ -124,12 +125,19 @@ func fetchAppVul() (bool, []*common.AppModuleVul) { var appVuls []*common.AppModuleVul for name, f := range appFetchers { + start := time.Now() + log.WithField("name", name).Info("Start app vulnerability fetcher") response, err := f.FetchUpdate() if err != nil { log.WithFields(log.Fields{"name": name, "error": err}).Error("App CVE update FAIL") return false, nil } else { appVuls = append(appVuls, response.Vulnerabilities...) + log.WithFields(log.Fields{ + "name": name, + "vulnerabilities": len(response.Vulnerabilities), + "elapsed": time.Since(start).String(), + }).Info("App vulnerability fetcher done") } } @@ -137,7 +145,11 @@ func fetchAppVul() (bool, []*common.AppModuleVul) { } func correctAppAffectedVersion(appVuls []*common.AppModuleVul) { - for _, app := range appVuls { + start := time.Now() + updated := 0 + checked := 0 + log.WithField("apps", len(appVuls)).Info("Start correcting app affected versions") + for i, app := range appVuls { if len(app.AffectedVer) == 0 || len(app.FixedVer) == 0 { if affects, fixes, ok := nvd.NVD.GetAffectedVersion(app.VulName); ok { // log.WithFields(log.Fields{"name": app.VulName, "affects": affects, "fixes": fixes}).Info("jar update") @@ -156,9 +168,24 @@ func correctAppAffectedVersion(appVuls []*common.AppModuleVul) { app.FixedVer = append(app.FixedVer, ver) } } + updated++ } } + checked++ + if (i+1)%1000 == 0 { + log.WithFields(log.Fields{ + "processed": i + 1, + "total": len(appVuls), + "updated": updated, + "elapsed": time.Since(start).String(), + }).Info("Correcting app affected versions progress") + } } + log.WithFields(log.Fields{ + "processed": checked, + "updated": updated, + "elapsed": time.Since(start).String(), + }).Info("Finished correcting app affected versions") } func fetchRawData() (bool, []*common.RawFile) { @@ -307,11 +334,16 @@ func fixSeverityScore(feedSeverity common.Priority, maxCVSSv2, maxCVSSv3 *common func assignMetadata(vuls []*common.Vulnerability, apps []*common.AppModuleVul) ([]*common.Vulnerability, []*common.AppModuleVul) { cveMap := make(map[string]*common.NVDMetadata) + start := time.Now() + log.WithFields(log.Fields{ + "distroVuls": len(vuls), + "appVuls": len(apps), + }).Info("Start assigning metadata") // Use two loops to cross-reference metadata provided by all feeds and nvd // first loop, for each cve merge meta with NVD - for _, v := range vuls { + for i, v := range vuls { cves := []common.CVE{common.CVE{Name: v.Name}} if len(v.CVEs) > 0 { cves = v.CVEs @@ -342,9 +374,18 @@ func assignMetadata(vuls []*common.Vulnerability, apps []*common.AppModuleVul) ( cveMap[key] = meta } } + if (i+1)%10000 == 0 { + log.WithFields(log.Fields{ + "phase": "distro-pass1", + "processed": i + 1, + "total": len(vuls), + "cveMap": len(cveMap), + "elapsed": time.Since(start).String(), + }).Info("Assign metadata progress") + } } - for _, app := range apps { + for i, app := range apps { cves := []string{app.VulName} if len(app.CVEs) > 0 { cves = append(cves, app.CVEs...) @@ -372,13 +413,22 @@ func assignMetadata(vuls []*common.Vulnerability, apps []*common.AppModuleVul) ( cveMap[cve] = meta } } + if (i+1)%1000 == 0 { + log.WithFields(log.Fields{ + "phase": "app-pass1", + "processed": i + 1, + "total": len(apps), + "cveMap": len(cveMap), + "elapsed": time.Since(start).String(), + }).Info("Assign metadata progress") + } } // second loop, assign the severity and score to the record outVuls := make([]*common.Vulnerability, 0) outApps := make([]*common.AppModuleVul, 0) - for _, v := range vuls { + for i, v := range vuls { cves := []common.CVE{common.CVE{Name: v.Name}} if len(v.CVEs) > 0 { cves = v.CVEs @@ -424,9 +474,18 @@ func assignMetadata(vuls []*common.Vulnerability, apps []*common.AppModuleVul) ( common.DEBUG_VULN(v, "post distro") } + if (i+1)%10000 == 0 { + log.WithFields(log.Fields{ + "phase": "distro-pass2", + "processed": i + 1, + "total": len(vuls), + "kept": len(outVuls), + "elapsed": time.Since(start).String(), + }).Info("Assign metadata progress") + } } - for _, app := range apps { + for i, app := range apps { cves := []string{app.VulName} if len(app.CVEs) > 0 { cves = append(cves, app.CVEs...) @@ -471,8 +530,24 @@ func assignMetadata(vuls []*common.Vulnerability, apps []*common.AppModuleVul) ( common.DEBUG_VULN(app, "post app") } + if (i+1)%1000 == 0 { + log.WithFields(log.Fields{ + "phase": "app-pass2", + "processed": i + 1, + "total": len(apps), + "kept": len(outApps), + "elapsed": time.Since(start).String(), + }).Info("Assign metadata progress") + } } + log.WithFields(log.Fields{ + "distroOut": len(outVuls), + "appOut": len(outApps), + "cveMap": len(cveMap), + "elapsed": time.Since(start).String(), + }).Info("Finished assigning metadata") + return outVuls, outApps } @@ -484,26 +559,36 @@ func fetch(datastore Datastore) (bool, []*common.Vulnerability, []*common.AppMod if !status { return status, nil, nil, nil } + log.WithField("distroVuls", len(osVuls)).Info("Fetched distro vulnerabilities") status, rawFiles := fetchRawData() if !status { return status, nil, nil, nil } + log.WithField("rawFiles", len(rawFiles)).Info("Fetched raw vulnerability files") status, appVuls := fetchAppVul() if !status { return status, nil, nil, nil } + log.WithField("appVuls", len(appVuls)).Info("Fetched app vulnerabilities") + log.Info("Start loading NVD metadata") if err := nvd.NVD.Load(); err != nil { log.Errorf("an error occured when loading NVD: %s.", err) return false, nil, nil, nil } + log.Info("Finished loading NVD metadata") appVuls = injectNvdWhitelistApps(appVuls) + log.WithField("appVuls", len(appVuls)).Info("Injected NVD whitelist apps") correctAppAffectedVersion(appVuls) vuls, apps := assignMetadata(osVuls, appVuls) + log.WithFields(log.Fields{ + "distroVuls": len(vuls), + "appVuls": len(apps), + }).Info("Fetch pipeline complete") return status, vuls, apps, rawFiles }