Skip to content

fix: improve oracle fetch url #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pohanhuang merged 1 commit into from
Jun 27, 2026
Merged

fix: improve oracle fetch url #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 145 additions & 23 deletions updater/fetchers/oracle/oracle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
package oracle

import (
"bufio"
"bytes"
"compress/bzip2"
"encoding/xml"
"fmt"
"io"
"net/http"
"regexp"
"sort"
"strconv"
"strings"
"time"
Expand All @@ -20,7 +22,7 @@ import (
const (
firstConsideredELSA = 7

ovalURI = "https://linux.oracle.com/oval/"
ovalURI = "https://linux.oracle.com/security/oval/"
retryTimes = 5
)

Expand All @@ -30,7 +32,7 @@ var (
".ksplice1.",
}

elsaRegexp = regexp.MustCompile(`com.oracle.elsa-(\d+).xml`)
elsaRegexp = regexp.MustCompile(`href="(com\.oracle\.elsa-(?:all|ol(?:6|7|8|9|10))\.xml\.bz2)"`)
)

type oval struct {
Expand Down Expand Up @@ -89,44 +91,47 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error)
log.Info("fetching Oracle vulnerabilities")

req, err := http.NewRequest("GET", ovalURI, nil)
if err != nil {
return resp, err
}
req.Header.Add("User-Agent", "dbgen")
client := http.Client{}
r, err := client.Do(req)
if err != nil {
log.Errorf("could not download Oracle's directory: %s", err)
return resp, common.ErrCouldNotDownload
}
defer r.Body.Close()

// Get the list of ELSAs that we have to process.
var elsaList []int
scanner := bufio.NewScanner(r.Body)
for scanner.Scan() {
line := scanner.Text()
r := elsaRegexp.FindStringSubmatch(line)
if len(r) == 2 && len(r[1]) > 4 {
year, _ := strconv.Atoi(r[1][:4])
if year >= common.FirstYear {
elsaNo, _ := strconv.Atoi(r[1])
elsaList = append(elsaList, elsaNo)
}
}
indexBody, err := io.ReadAll(r.Body)
if err != nil {
log.Errorf("could not read Oracle's directory: %s", err)
return resp, common.ErrCouldNotDownload
}

feedFiles := listFeedFiles(indexBody)
if len(feedFiles) == 0 {
log.Error("Failed to find Oracle oval feeds")
return resp, fmt.Errorf("Failed to find Oracle oval feeds")
}

r.Body.Close()
vulnMap := make(map[string]common.Vulnerability)

for i, elsa := range elsaList {
for i, elsaFile := range feedFiles {
var vs []common.Vulnerability

retry := 0
for retry <= retryTimes {
elsaFile := fmt.Sprintf("com.oracle.elsa-%d.xml", elsa)
rurl := fmt.Sprintf("%s%s", ovalURI, elsaFile)

client := http.Client{}
req, _ := http.NewRequest("GET", rurl, nil)
req, err := http.NewRequest("GET", rurl, nil)
if err != nil {
return resp, err
}
req.Header.Add("User-Agent", "dbgen")
if r, err := client.Do(req); err == nil {
vs, err = parseELSA(elsaFile, r.Body)
vs, err = parseBZ2ELSA(elsaFile, r.Body)

r.Body.Close()
if err == nil {
Expand All @@ -148,7 +153,7 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error)

// Collect vulnerabilities.
for _, v := range vs {
resp.Vulnerabilities = append(resp.Vulnerabilities, v)
mergeVulnerability(vulnMap, v)
}

// Pause to prevent the website from blacklisting us.
Expand All @@ -157,6 +162,10 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error)
}
}

for _, v := range vulnMap {
resp.Vulnerabilities = append(resp.Vulnerabilities, v)
}

if len(resp.Vulnerabilities) == 0 {
log.Error("Failed to fetch Oracle oval feed")
return resp, fmt.Errorf("Failed to fetch Oracle oval feed")
Expand All @@ -166,11 +175,30 @@ func (f *OracleFetcher) FetchUpdate() (resp updater.FetcherResponse, err error)
return resp, nil
}

func parseBZ2ELSA(elsa string, compressedReader io.Reader) ([]common.Vulnerability, error) {
return parseELSA(elsa, bzip2.NewReader(compressedReader))
}

func parseELSA(elsa string, ovalReader io.Reader) (vulnerabilities []common.Vulnerability, err error) {
body, err := io.ReadAll(ovalReader)
if err != nil {
return nil, err
}

trimmed := bytes.TrimSpace(body)
if bytes.HasPrefix(trimmed, []byte("<!DOCTYPE html")) || bytes.HasPrefix(trimmed, []byte("<html")) {
log.WithFields(log.Fields{"elsa": elsa}).Warn("Oracle ELSA returned HTML instead of XML, skipping")
return nil, nil
}

// Decode the XML.
var ov oval
err = xml.NewDecoder(ovalReader).Decode(&ov)
err = xml.NewDecoder(bytes.NewReader(body)).Decode(&ov)
if err != nil {
if bytes.Contains(bytes.ToLower(trimmed), []byte("<html")) || bytes.Contains(bytes.ToLower(trimmed), []byte("<body")) {
log.WithFields(log.Fields{"elsa": elsa}).Warn("Oracle ELSA returned non-XML content, skipping")
return nil, nil
}
log.Errorf("could not decode Oracle's XML: %s", err)
err = common.ErrCouldNotParse
return
Expand Down Expand Up @@ -218,6 +246,100 @@ func parseELSA(elsa string, ovalReader io.Reader) (vulnerabilities []common.Vuln
return
}

func listFeedFiles(indexBody []byte) []string {
matches := elsaRegexp.FindAllSubmatch(indexBody, -1)
files := make([]string, 0, len(matches))
seen := make(map[string]struct{}, len(matches))
for _, match := range matches {
if len(match) != 2 {
continue
}
name := string(match[1])
if _, ok := seen[name]; ok {
continue
}
seen[name] = struct{}{}
files = append(files, name)
}

sort.Strings(files)
return files
}

func mergeVulnerability(vulnMap map[string]common.Vulnerability, v common.Vulnerability) {
if existing, ok := vulnMap[v.Name]; ok {
existing.FixedIn = mergeFeatureVersions(existing.FixedIn, v.FixedIn)
existing.CVEs = mergeCVEs(existing.CVEs, v.CVEs)
if existing.Description == "" {
existing.Description = v.Description
}
if existing.Link == "" {
existing.Link = v.Link
}
if existing.Severity == common.Unknown {
existing.Severity = v.Severity
}
if existing.IssuedDate.IsZero() || (!v.IssuedDate.IsZero() && v.IssuedDate.Before(existing.IssuedDate)) {
existing.IssuedDate = v.IssuedDate
}
if existing.LastModDate.IsZero() || v.LastModDate.After(existing.LastModDate) {
existing.LastModDate = v.LastModDate
}
vulnMap[v.Name] = existing
return
}

v.FixedIn = mergeFeatureVersions(nil, v.FixedIn)
v.CVEs = mergeCVEs(nil, v.CVEs)
vulnMap[v.Name] = v
}

func mergeFeatureVersions(existing []common.FeatureVersion, incoming []common.FeatureVersion) []common.FeatureVersion {
merged := make([]common.FeatureVersion, 0, len(existing)+len(incoming))
seen := make(map[string]struct{}, len(existing)+len(incoming))

for _, fv := range existing {
key := fv.Feature.Namespace + ":" + fv.Feature.Name + ":" + fv.Version.String()
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
merged = append(merged, fv)
}
for _, fv := range incoming {
key := fv.Feature.Namespace + ":" + fv.Feature.Name + ":" + fv.Version.String()
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
merged = append(merged, fv)
}

return merged
}

func mergeCVEs(existing []common.CVE, incoming []common.CVE) []common.CVE {
merged := make([]common.CVE, 0, len(existing)+len(incoming))
seen := make(map[string]struct{}, len(existing)+len(incoming))

for _, cve := range existing {
if _, ok := seen[cve.Name]; ok {
continue
}
seen[cve.Name] = struct{}{}
merged = append(merged, cve)
}
for _, cve := range incoming {
if _, ok := seen[cve.Name]; ok {
continue
}
seen[cve.Name] = struct{}{}
merged = append(merged, cve)
}

return merged
}

func getCriterions(node criteria) [][]criterion {
// Filter useless criterions.
var criterions []criterion
Expand Down
87 changes: 87 additions & 0 deletions updater/fetchers/oracle/oracle_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package oracle

import (
"strings"
"testing"

"github.com/vul-dbgen/common"
)

func TestParseELSASkipsHTMLResponse(t *testing.T) {
html := `
<!DOCTYPE html>
<html>
<body>
<hr>
</body>
</html>`

vuls, err := parseELSA("com.oracle.elsa-20269999.xml", strings.NewReader(html))
if err != nil {
t.Fatalf("expected HTML response to be skipped without error, got: %v", err)
}
if len(vuls) != 0 {
t.Fatalf("expected no vulnerabilities for skipped HTML response, got: %d", len(vuls))
}
}

func TestListFeedFiles(t *testing.T) {
indexHTML := `
<tr><td><a href="com.oracle.elsa-all.xml.bz2">com.oracle.elsa-all...&gt;</a></td></tr>
<tr><td><a href="com.oracle.elsa-ol6.xml.bz2">com.oracle.elsa-ol6...&gt;</a></td></tr>
<tr><td><a href="com.oracle.elsa-ol7.xml.bz2">com.oracle.elsa-ol7...&gt;</a></td></tr>
<tr><td><a href="com.oracle.elsa-ol8.xml.bz2">com.oracle.elsa-ol8...&gt;</a></td></tr>
<tr><td><a href="com.oracle.elsa-ol9.xml.bz2">com.oracle.elsa-ol9...&gt;</a></td></tr>
<tr><td><a href="com.oracle.elsa-ol10.xml.bz2">com.oracle.elsa-ol10...&gt;</a></td></tr>
<tr><td><a href="com.oracle.elsa-2026.xml.bz2">com.oracle.elsa-2026...&gt;</a></td></tr>`

files := listFeedFiles([]byte(indexHTML))
expected := []string{
"com.oracle.elsa-all.xml.bz2",
"com.oracle.elsa-ol10.xml.bz2",
"com.oracle.elsa-ol6.xml.bz2",
"com.oracle.elsa-ol7.xml.bz2",
"com.oracle.elsa-ol8.xml.bz2",
"com.oracle.elsa-ol9.xml.bz2",
}

if len(files) != len(expected) {
t.Fatalf("expected %d files, got %d: %v", len(expected), len(files), files)
}
for i := range expected {
if files[i] != expected[i] {
t.Fatalf("expected file %d to be %q, got %q", i, expected[i], files[i])
}
}
}

func TestMergeVulnerabilityDeduplicatesFixedInAndCVEs(t *testing.T) {
vulnMap := make(map[string]common.Vulnerability)
version, err := common.NewVersion("1.1.1k-1")
if err != nil {
t.Fatalf("failed to parse test version: %v", err)
}
fixedIn := common.FeatureVersion{
Feature: common.Feature{
Name: "openssl",
Namespace: "oracle:9",
},
Version: version,
}
v := common.Vulnerability{
Name: "ELSA-2026-0001",
FixedIn: []common.FeatureVersion{fixedIn},
CVEs: []common.CVE{{Name: "CVE-2026-0001"}},
}

mergeVulnerability(vulnMap, v)
mergeVulnerability(vulnMap, v)

got := vulnMap[v.Name]
if len(got.FixedIn) != 1 {
t.Fatalf("expected one fixed-in entry after dedupe, got %d", len(got.FixedIn))
}
if len(got.CVEs) != 1 {
t.Fatalf("expected one CVE after dedupe, got %d", len(got.CVEs))
}
}
Loading
Loading