From 932b7569ea9b2bfbd6ce7b67d095e739969e45d5 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:23:51 -0600 Subject: [PATCH 1/2] perf(qualys): look up QID details once per report, not once per finding The Qualys XML parser found each finding's QID details with a search from the document root, which walks every host before it reaches the glossary. Parsing cost findings x report size: a 56-host, 2,408-finding report took 40.0 s of CPU to parse, against 2.0 s for 7 hosts. The glossary is now indexed once per report and each finding reads its entry from that index. The first entry for an id still wins, as the search returned it, and the parsed findings are identical to before on reports of 7 to 56 hosts. Co-Authored-By: Claude Opus 5.5 --- dojo/tools/qualys/parser.py | 22 ++++++++-- unittests/tools/test_qualys_parser.py | 58 +++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/dojo/tools/qualys/parser.py b/dojo/tools/qualys/parser.py index 465301310f4..ad1072a7d32 100644 --- a/dojo/tools/qualys/parser.py +++ b/dojo/tools/qualys/parser.py @@ -201,7 +201,7 @@ def split_cvss(value, _temp): _temp["CVSS_value"] = float(value) -def parse_finding(host, tree): +def parse_finding(host, glossary): ret_rows = [] issue_row = {} @@ -265,8 +265,7 @@ def parse_finding(host, tree): # DefectDojo does not support cvssv2 temp["CVSS_vector"] = None - search = f".//GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS[@id='{gid}']" - vuln_item = tree.find(search) + vuln_item = glossary.get(gid) if vuln_item is not None: finding = Finding() # Vuln name @@ -373,6 +372,21 @@ def qualys_parser(qualys_xml_file): host_list = tree.find("HOST_LIST") finding_list = [] if host_list is not None: + glossary = vuln_details_by_id(tree) for host in host_list: - finding_list += parse_finding(host, tree) + finding_list += parse_finding(host, glossary) return finding_list + + +def vuln_details_by_id(tree): + """ + The report's glossary entries by QID, read once per report. + + Looking a QID up with a search from the document root walks every host before it reaches the + glossary, so doing that for each finding made a parse cost findings x report size. The first + entry for an id wins, as that search returned it. + """ + glossary = {} + for vuln_item in tree.iterfind(".//GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS"): + glossary.setdefault(vuln_item.get("id"), vuln_item) + return glossary diff --git a/unittests/tools/test_qualys_parser.py b/unittests/tools/test_qualys_parser.py index 060b6b9fcc0..65e612fc568 100644 --- a/unittests/tools/test_qualys_parser.py +++ b/unittests/tools/test_qualys_parser.py @@ -1,4 +1,8 @@ import datetime +import io +import re +from unittest import mock +from xml.etree import ElementTree as ET from django.test import override_settings @@ -7,8 +11,62 @@ from unittests.dojo_test_case import DojoTestCase, get_unit_tests_scans_path +def _report_with_hosts(count): + """The sample report with its hosts repeated to ``count`` hosts, each with its own address.""" + sample = (get_unit_tests_scans_path("qualys") / "Qualys_Sample_Report.xml").read_text(encoding="utf-8") + head, rest = sample.split("", 1) + hosts, tail = rest.split("", 1) + blocks = re.findall(r".*?", hosts, flags=re.DOTALL) + repeated = [ + re.sub(r"[^<]*", f"192.0.2.{index + 1}", blocks[index % len(blocks)], count=1) + for index in range(count) + ] + return _named(f"{head}{''.join(repeated)}{tail}") + + +def _named(content): + report = io.BytesIO(content.encode()) + report.name = "report.xml" + return report + + class TestQualysParser(DojoTestCase): + def test_glossary_is_searched_once_per_report_not_once_per_finding(self): + # Every finding used to look its QID up with a search from the document root, which walks + # all the hosts before it reaches the glossary: the parse was findings x document size. + searches = [] + for method in ("find", "findall", "iterfind"): + original = getattr(ET.ElementTree, method) + + def counting(self, path, namespaces=None, *, _original=original): + if "GLOSSARY" in path: + searches.append(path) + return _original(self, path, namespaces) + + patcher = mock.patch.object(ET.ElementTree, method, counting) + patcher.start() + self.addCleanup(patcher.stop) + + small = QualysParser().get_findings(_report_with_hosts(2), Test()) + small_searches = len(searches) + searches.clear() + large = QualysParser().get_findings(_report_with_hosts(14), Test()) + + self.assertGreater(len(large), len(small)) + self.assertEqual(small_searches, len(searches)) + + def test_repeated_glossary_entry_keeps_the_first(self): + sample = (get_unit_tests_scans_path("qualys") / "Qualys_Sample_Report.xml").read_text(encoding="utf-8") + details = re.search(r".*?", sample, flags=re.DOTALL).group(0) + shadow = re.sub(r"<!\[CDATA\[[^\]]*\]\]>", "<![CDATA[Shadowed]]>", details, count=1) + report = _named(sample.replace("", f"{shadow}", 1)) + + titles = {finding.title for finding in QualysParser().get_findings(report, Test())} + + self.assertIn("QID-6 | DNS Host Name", titles) + self.assertNotIn("QID-6 | Shadowed", titles) + @override_settings(USE_FIRST_SEEN=True) def test_parse_file_with_no_vuln_has_no_findings_first_seen(self): self.parse_file_with_no_vuln_has_no_findings() From e26b5e08f7887907b2633a968181a14fd5dde9a7 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:16:09 -0600 Subject: [PATCH 2/2] test(qualys): count per-QID lookups wherever the search starts The guard only watched searches made on the whole tree, so a lookup made from the root element once per finding would have brought the quadratic parse back without failing it. Every find, findall, findtext and iterfind, on a tree or an element, resolves its path through ElementPath.iterfind, so that is what the test counts now. Co-Authored-By: Claude Opus 5.5 --- unittests/tools/test_qualys_parser.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/unittests/tools/test_qualys_parser.py b/unittests/tools/test_qualys_parser.py index 65e612fc568..814b6ae12fe 100644 --- a/unittests/tools/test_qualys_parser.py +++ b/unittests/tools/test_qualys_parser.py @@ -2,7 +2,7 @@ import io import re from unittest import mock -from xml.etree import ElementTree as ET +from xml.etree import ElementPath from django.test import override_settings @@ -35,18 +35,19 @@ class TestQualysParser(DojoTestCase): def test_glossary_is_searched_once_per_report_not_once_per_finding(self): # Every finding used to look its QID up with a search from the document root, which walks # all the hosts before it reaches the glossary: the parse was findings x document size. + # Every find, findall, findtext and iterfind, on an element or on the whole tree, resolves its + # path through ElementPath.iterfind, so a per-QID lookup from any starting point is counted. searches = [] - for method in ("find", "findall", "iterfind"): - original = getattr(ET.ElementTree, method) + original = ElementPath.iterfind - def counting(self, path, namespaces=None, *, _original=original): - if "GLOSSARY" in path: - searches.append(path) - return _original(self, path, namespaces) + def counting(elem, path, namespaces=None): + if "VULN_DETAILS" in path: + searches.append(path) + return original(elem, path, namespaces) - patcher = mock.patch.object(ET.ElementTree, method, counting) - patcher.start() - self.addCleanup(patcher.stop) + patcher = mock.patch.object(ElementPath, "iterfind", counting) + patcher.start() + self.addCleanup(patcher.stop) small = QualysParser().get_findings(_report_with_hosts(2), Test()) small_searches = len(searches)