From 64379122de2274c4a3417a4f2fe35edd621a1287 Mon Sep 17 00:00:00 2001 From: Muntasir Mamun Date: Thu, 22 Oct 2026 20:09:27 +0600 Subject: [PATCH] [ci] Create annotations for failed ctests/gtests in pull requests Fixes #18475 Parse failed test blocks from CTest logs (LastTestsFailed.log and LastTest.log) and emit GitHub Actions ::error annotations mapping failing tests directly to the relevant source file and line (e.g. gtest failures, macro errors, and Python tracebacks in roottest). Exclude Windows to prevent duplicate annotations on test reruns. --- .../workflows/root-ci-config/build_root.py | 3 + .../workflows/root-ci-config/build_utils.py | 123 +++++++++++++++++- .../root-ci-config/test_annotate_failures.py | 106 +++++++++++++++ .github/workflows/test-ci-scripts.yml | 4 +- 4 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/root-ci-config/test_annotate_failures.py diff --git a/.github/workflows/root-ci-config/build_root.py b/.github/workflows/root-ci-config/build_root.py index 7ae789e70baf6..13876d5c8f9ac 100755 --- a/.github/workflows/root-ci-config/build_root.py +++ b/.github/workflows/root-ci-config/build_root.py @@ -220,6 +220,9 @@ def main(): print_trace() def handle_test_failure(ctest_returncode): + if not WINDOWS: + build_utils.annotate_ctest_failures(WORKDIR) + logloc = os.path.join(WORKDIR, "build", "Testing", "Temporary", "LastTestsFailed.log") if os.path.isfile(logloc): with open(logloc, 'r') as logf: diff --git a/.github/workflows/root-ci-config/build_utils.py b/.github/workflows/root-ci-config/build_utils.py index 5ace08990c732..fe15627332e90 100755 --- a/.github/workflows/root-ci-config/build_utils.py +++ b/.github/workflows/root-ci-config/build_utils.py @@ -8,11 +8,12 @@ import sys import textwrap import time +import re from functools import wraps from hashlib import sha1 from http import HTTPStatus from shutil import which -from typing import Callable, Dict +from typing import Callable, Dict, List, Optional dry_run : bool = False @@ -360,3 +361,123 @@ def remove_file_match_ext(directory: str, extension: str) -> str: os.remove(file_path) count += 1 print_fancy(f"Deleted {count} gcda files") + + +def escape_workflow_data(s: str) -> str: + """Escapes characters for GitHub Actions workflow command messages.""" + return s.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def extract_ctest_annotations(last_failed_log_content: str, last_test_log_content: str, srcdir: str = "") -> List[Dict[str, Optional[str]]]: + """ + Parses failed ctests/gtests from LastTestsFailed.log and LastTest.log. + Returns a list of dictionaries with keys: file, line, title, message. + """ + annotations = [] + failed_tests = set() + + for line in last_failed_log_content.splitlines(): + parts = line.strip().split(":", 1) + if len(parts) == 2 and parts[1].strip(): + failed_tests.add(parts[1].strip()) + + if not failed_tests: + return annotations + + patterns = [ + # gtest Failure: /path/file.cxx:88: Failure + re.compile(r'([^\s:\n]+\.(?:cxx|cpp|cc|c|C|h|hpp|py)):(\d+):\s*Failure\b[^\n]*\n?([^\n]*)'), + # compiler / macro error: /path/file.C:42:15: error: ... + re.compile(r'([^\s:\n]+\.(?:cxx|cpp|cc|c|C|h|hpp|py)):(\d+):(?:\d+:)?\s*(?:fatal error|error):\s*([^\n]+)'), + # Python traceback: File "/path/file.py", line 15, in ... + re.compile(r'File "([^"\n]+\.py)", line (\d+)(?:, in [^\n]+)?\n(?:\s*[^\n]+\n)*\s*([A-Za-z0-9_.]*(?:Error|Exception):[^\n]+)'), + re.compile(r'File "([^"\n]+\.py)", line (\d+)(?:, in [^\n]+)?\n\s*([^\n]*)'), + # Generic assertion failure: file:line: assertion failed / failed + re.compile(r'([^\s:\n]+\.(?:cxx|cpp|cc|c|C|h|hpp|py)):(\d+):\s*([^\n]+)'), + ] + + # Find test blocks in LastTest.log + test_blocks = re.findall( + r'Testing:\s*(\S+)\s*\n.*?-{30,}\n(.*?)\n-{30,}\s*\n.*?Test Failed', + last_test_log_content, + re.DOTALL + ) + + found_tests = set() + for test_name, output in test_blocks: + if test_name not in failed_tests: + continue + found_tests.add(test_name) + + matched = False + for pattern in patterns: + match = pattern.search(output) + if match: + file_path, line, msg = match.group(1), match.group(2), match.group(3) + if srcdir and file_path.startswith(srcdir): + file_path = os.path.relpath(file_path, srcdir) + elif "/src/" in file_path: + file_path = file_path.split("/src/", 1)[1] + + msg = msg.strip() or f"Test {test_name} failed" + annotations.append({ + "file": file_path, + "line": line, + "title": test_name, + "message": msg, + }) + matched = True + break + + if not matched: + annotations.append({ + "file": None, + "line": None, + "title": test_name, + "message": f"Test {test_name} failed. Check test log for details.", + }) + + for remaining_test in failed_tests - found_tests: + annotations.append({ + "file": None, + "line": None, + "title": remaining_test, + "message": f"Test {remaining_test} failed. Check test log for details.", + }) + + return annotations + + +def annotate_ctest_failures(workdir: str) -> None: + """ + Emits GitHub Actions annotations for failed ctests/gtests so they appear + directly in the 'Files changed' tab of pull requests. + """ + builddir = os.path.join(workdir, "build") + srcdir = os.path.join(workdir, "src") + last_failed_path = os.path.join(builddir, "Testing", "Temporary", "LastTestsFailed.log") + last_test_path = os.path.join(builddir, "Testing", "Temporary", "LastTest.log") + + if not os.path.isfile(last_failed_path) or not os.path.isfile(last_test_path): + return + + try: + with open(last_failed_path, "r", encoding="utf-8", errors="replace") as f: + last_failed_content = f.read() + with open(last_test_path, "r", encoding="utf-8", errors="replace") as f: + last_test_content = f.read() + except OSError: + return + + annotations = extract_ctest_annotations(last_failed_content, last_test_content, srcdir) + if annotations: + print_fancy("\n::group::CTest Failure Annotations (GitHub Actions)", sgr=1) + for ann in annotations: + msg = escape_workflow_data(ann["message"]) + title = escape_workflow_data(ann["title"]) + if ann["file"] and ann["line"]: + print(f"::error file={ann['file']},line={ann['line']},title={title}::{msg}") + else: + print(f"::error title={title}::{msg}") + print("::endgroup::\n") + diff --git a/.github/workflows/root-ci-config/test_annotate_failures.py b/.github/workflows/root-ci-config/test_annotate_failures.py new file mode 100644 index 0000000000000..0739efb4d0a29 --- /dev/null +++ b/.github/workflows/root-ci-config/test_annotate_failures.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +"""Unit tests for CTest failure annotation logic in build_utils.py. + +Usage: + + python3 test_annotate_failures.py +""" + +import os +import tempfile +import unittest + +import build_utils + + +class AnnotateCTestFailuresTest(unittest.TestCase): + def test_extract_annotations_gtest(self): + failed_log = "1:gtest-core-foundation\n" + test_log = """ +1/1 Testing: gtest-core-foundation +---------------------------------------------------------- +/ROOT-CI/src/core/foundation/test/TStringTest.cxx:88: Failure +Value of: str.Length() +Actual: 0 +Expected: 5 +---------------------------------------------------------- +Test Failed. +""" + annotations = build_utils.extract_ctest_annotations(failed_log, test_log, "/ROOT-CI/src") + self.assertEqual(len(annotations), 1) + self.assertEqual(annotations[0]["file"], "core/foundation/test/TStringTest.cxx") + self.assertEqual(annotations[0]["line"], "88") + self.assertEqual(annotations[0]["title"], "gtest-core-foundation") + self.assertIn("Value of: str.Length()", annotations[0]["message"]) + + def test_extract_annotations_compiler_macro(self): + failed_log = "2:roottest-root-meta-test\n" + test_log = """ +1/1 Testing: roottest-root-meta-test +---------------------------------------------------------- +/ROOT-CI/src/roottest/root/meta/test.C:42:15: error: use of undeclared identifier 'foo' +---------------------------------------------------------- +Test Failed. +""" + annotations = build_utils.extract_ctest_annotations(failed_log, test_log, "/ROOT-CI/src") + self.assertEqual(len(annotations), 1) + self.assertEqual(annotations[0]["file"], "roottest/root/meta/test.C") + self.assertEqual(annotations[0]["line"], "42") + self.assertEqual(annotations[0]["title"], "roottest-root-meta-test") + self.assertIn("use of undeclared identifier 'foo'", annotations[0]["message"]) + + def test_extract_annotations_python_traceback(self): + failed_log = "3:roottest-python-import\n" + test_log = """ +1/1 Testing: roottest-python-import +---------------------------------------------------------- +Traceback (most recent call last): + File "/ROOT-CI/src/bindings/pyroot/test/test_import.py", line 15, in test_something + self.assertEqual(a, b) +AssertionError: 1 != 2 +---------------------------------------------------------- +Test Failed. +""" + annotations = build_utils.extract_ctest_annotations(failed_log, test_log, "/ROOT-CI/src") + self.assertEqual(len(annotations), 1) + self.assertEqual(annotations[0]["file"], "bindings/pyroot/test/test_import.py") + self.assertEqual(annotations[0]["line"], "15") + self.assertEqual(annotations[0]["title"], "roottest-python-import") + self.assertIn("AssertionError: 1 != 2", annotations[0]["message"]) + + def test_extract_annotations_unknown_failure(self): + failed_log = "4:roottest-mystery-crash\n" + test_log = """ +1/1 Testing: roottest-mystery-crash +---------------------------------------------------------- +Process killed by signal 11 +---------------------------------------------------------- +Test Failed. +""" + annotations = build_utils.extract_ctest_annotations(failed_log, test_log, "/ROOT-CI/src") + self.assertEqual(len(annotations), 1) + self.assertIsNone(annotations[0]["file"]) + self.assertIsNone(annotations[0]["line"]) + self.assertEqual(annotations[0]["title"], "roottest-mystery-crash") + + def test_annotate_ctest_failures_end_to_end(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_build = os.path.join(tmpdir, "build", "Testing", "Temporary") + os.makedirs(temp_build) + with open(os.path.join(temp_build, "LastTestsFailed.log"), "w", encoding="utf-8") as f: + f.write("1:test-sample\n") + with open(os.path.join(temp_build, "LastTest.log"), "w", encoding="utf-8") as f: + f.write(""" +1/1 Testing: test-sample +---------------------------------------------------------- +/dummy/src/roottest/sample.C:10: error: failed assertion +---------------------------------------------------------- +Test Failed. +""") + # Should run without error + build_utils.annotate_ctest_failures(tmpdir) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/test-ci-scripts.yml b/.github/workflows/test-ci-scripts.yml index 71992410bcb0a..71c9f58af0fd2 100644 --- a/.github/workflows/test-ci-scripts.yml +++ b/.github/workflows/test-ci-scripts.yml @@ -15,4 +15,6 @@ jobs: uses: actions/checkout@v6 - name: Run unit tests - run: python3 .github/workflows/root-ci-config/test_null_build_check.py --verbose + run: | + python3 .github/workflows/root-ci-config/test_null_build_check.py --verbose + python3 .github/workflows/root-ci-config/test_annotate_failures.py --verbose