diff --git a/.github/workflows/root-ci-config/build_root.py b/.github/workflows/root-ci-config/build_root.py index 7ae789e70baf6..13876d5c8f9ac 100755 --- a/.github/workflows/root-ci-config/build_root.py +++ b/.github/workflows/root-ci-config/build_root.py @@ -220,6 +220,9 @@ def main(): print_trace() def handle_test_failure(ctest_returncode): + if not WINDOWS: + build_utils.annotate_ctest_failures(WORKDIR) + logloc = os.path.join(WORKDIR, "build", "Testing", "Temporary", "LastTestsFailed.log") if os.path.isfile(logloc): with open(logloc, 'r') as logf: diff --git a/.github/workflows/root-ci-config/build_utils.py b/.github/workflows/root-ci-config/build_utils.py index 5ace08990c732..fe15627332e90 100755 --- a/.github/workflows/root-ci-config/build_utils.py +++ b/.github/workflows/root-ci-config/build_utils.py @@ -8,11 +8,12 @@ import sys import textwrap import time +import re from functools import wraps from hashlib import sha1 from http import HTTPStatus from shutil import which -from typing import Callable, Dict +from typing import Callable, Dict, List, Optional dry_run : bool = False @@ -360,3 +361,123 @@ def remove_file_match_ext(directory: str, extension: str) -> str: os.remove(file_path) count += 1 print_fancy(f"Deleted {count} gcda files") + + +def escape_workflow_data(s: str) -> str: + """Escapes characters for GitHub Actions workflow command messages.""" + return s.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def extract_ctest_annotations(last_failed_log_content: str, last_test_log_content: str, srcdir: str = "") -> List[Dict[str, Optional[str]]]: + """ + Parses failed ctests/gtests from LastTestsFailed.log and LastTest.log. + Returns a list of dictionaries with keys: file, line, title, message. + """ + annotations = [] + failed_tests = set() + + for line in last_failed_log_content.splitlines(): + parts = line.strip().split(":", 1) + if len(parts) == 2 and parts[1].strip(): + failed_tests.add(parts[1].strip()) + + if not failed_tests: + return annotations + + patterns = [ + # gtest Failure: /path/file.cxx:88: Failure + re.compile(r'([^\s:\n]+\.(?:cxx|cpp|cc|c|C|h|hpp|py)):(\d+):\s*Failure\b[^\n]*\n?([^\n]*)'), + # compiler / macro error: /path/file.C:42:15: error: ... + re.compile(r'([^\s:\n]+\.(?:cxx|cpp|cc|c|C|h|hpp|py)):(\d+):(?:\d+:)?\s*(?:fatal error|error):\s*([^\n]+)'), + # Python traceback: File "/path/file.py", line 15, in ... + re.compile(r'File "([^"\n]+\.py)", line (\d+)(?:, in [^\n]+)?\n(?:\s*[^\n]+\n)*\s*([A-Za-z0-9_.]*(?:Error|Exception):[^\n]+)'), + re.compile(r'File "([^"\n]+\.py)", line (\d+)(?:, in [^\n]+)?\n\s*([^\n]*)'), + # Generic assertion failure: file:line: assertion failed / failed + re.compile(r'([^\s:\n]+\.(?:cxx|cpp|cc|c|C|h|hpp|py)):(\d+):\s*([^\n]+)'), + ] + + # Find test blocks in LastTest.log + test_blocks = re.findall( + r'Testing:\s*(\S+)\s*\n.*?-{30,}\n(.*?)\n-{30,}\s*\n.*?Test Failed', + last_test_log_content, + re.DOTALL + ) + + found_tests = set() + for test_name, output in test_blocks: + if test_name not in failed_tests: + continue + found_tests.add(test_name) + + matched = False + for pattern in patterns: + match = pattern.search(output) + if match: + file_path, line, msg = match.group(1), match.group(2), match.group(3) + if srcdir and file_path.startswith(srcdir): + file_path = os.path.relpath(file_path, srcdir) + elif "/src/" in file_path: + file_path = file_path.split("/src/", 1)[1] + + msg = msg.strip() or f"Test {test_name} failed" + annotations.append({ + "file": file_path, + "line": line, + "title": test_name, + "message": msg, + }) + matched = True + break + + if not matched: + annotations.append({ + "file": None, + "line": None, + "title": test_name, + "message": f"Test {test_name} failed. Check test log for details.", + }) + + for remaining_test in failed_tests - found_tests: + annotations.append({ + "file": None, + "line": None, + "title": remaining_test, + "message": f"Test {remaining_test} failed. Check test log for details.", + }) + + return annotations + + +def annotate_ctest_failures(workdir: str) -> None: + """ + Emits GitHub Actions annotations for failed ctests/gtests so they appear + directly in the 'Files changed' tab of pull requests. + """ + builddir = os.path.join(workdir, "build") + srcdir = os.path.join(workdir, "src") + last_failed_path = os.path.join(builddir, "Testing", "Temporary", "LastTestsFailed.log") + last_test_path = os.path.join(builddir, "Testing", "Temporary", "LastTest.log") + + if not os.path.isfile(last_failed_path) or not os.path.isfile(last_test_path): + return + + try: + with open(last_failed_path, "r", encoding="utf-8", errors="replace") as f: + last_failed_content = f.read() + with open(last_test_path, "r", encoding="utf-8", errors="replace") as f: + last_test_content = f.read() + except OSError: + return + + annotations = extract_ctest_annotations(last_failed_content, last_test_content, srcdir) + if annotations: + print_fancy("\n::group::CTest Failure Annotations (GitHub Actions)", sgr=1) + for ann in annotations: + msg = escape_workflow_data(ann["message"]) + title = escape_workflow_data(ann["title"]) + if ann["file"] and ann["line"]: + print(f"::error file={ann['file']},line={ann['line']},title={title}::{msg}") + else: + print(f"::error title={title}::{msg}") + print("::endgroup::\n") + diff --git a/.github/workflows/root-ci-config/test_annotate_failures.py b/.github/workflows/root-ci-config/test_annotate_failures.py new file mode 100644 index 0000000000000..0739efb4d0a29 --- /dev/null +++ b/.github/workflows/root-ci-config/test_annotate_failures.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +"""Unit tests for CTest failure annotation logic in build_utils.py. + +Usage: + + python3 test_annotate_failures.py +""" + +import os +import tempfile +import unittest + +import build_utils + + +class AnnotateCTestFailuresTest(unittest.TestCase): + def test_extract_annotations_gtest(self): + failed_log = "1:gtest-core-foundation\n" + test_log = """ +1/1 Testing: gtest-core-foundation +---------------------------------------------------------- +/ROOT-CI/src/core/foundation/test/TStringTest.cxx:88: Failure +Value of: str.Length() +Actual: 0 +Expected: 5 +---------------------------------------------------------- +Test Failed. +""" + annotations = build_utils.extract_ctest_annotations(failed_log, test_log, "/ROOT-CI/src") + self.assertEqual(len(annotations), 1) + self.assertEqual(annotations[0]["file"], "core/foundation/test/TStringTest.cxx") + self.assertEqual(annotations[0]["line"], "88") + self.assertEqual(annotations[0]["title"], "gtest-core-foundation") + self.assertIn("Value of: str.Length()", annotations[0]["message"]) + + def test_extract_annotations_compiler_macro(self): + failed_log = "2:roottest-root-meta-test\n" + test_log = """ +1/1 Testing: roottest-root-meta-test +---------------------------------------------------------- +/ROOT-CI/src/roottest/root/meta/test.C:42:15: error: use of undeclared identifier 'foo' +---------------------------------------------------------- +Test Failed. +""" + annotations = build_utils.extract_ctest_annotations(failed_log, test_log, "/ROOT-CI/src") + self.assertEqual(len(annotations), 1) + self.assertEqual(annotations[0]["file"], "roottest/root/meta/test.C") + self.assertEqual(annotations[0]["line"], "42") + self.assertEqual(annotations[0]["title"], "roottest-root-meta-test") + self.assertIn("use of undeclared identifier 'foo'", annotations[0]["message"]) + + def test_extract_annotations_python_traceback(self): + failed_log = "3:roottest-python-import\n" + test_log = """ +1/1 Testing: roottest-python-import +---------------------------------------------------------- +Traceback (most recent call last): + File "/ROOT-CI/src/bindings/pyroot/test/test_import.py", line 15, in test_something + self.assertEqual(a, b) +AssertionError: 1 != 2 +---------------------------------------------------------- +Test Failed. +""" + annotations = build_utils.extract_ctest_annotations(failed_log, test_log, "/ROOT-CI/src") + self.assertEqual(len(annotations), 1) + self.assertEqual(annotations[0]["file"], "bindings/pyroot/test/test_import.py") + self.assertEqual(annotations[0]["line"], "15") + self.assertEqual(annotations[0]["title"], "roottest-python-import") + self.assertIn("AssertionError: 1 != 2", annotations[0]["message"]) + + def test_extract_annotations_unknown_failure(self): + failed_log = "4:roottest-mystery-crash\n" + test_log = """ +1/1 Testing: roottest-mystery-crash +---------------------------------------------------------- +Process killed by signal 11 +---------------------------------------------------------- +Test Failed. +""" + annotations = build_utils.extract_ctest_annotations(failed_log, test_log, "/ROOT-CI/src") + self.assertEqual(len(annotations), 1) + self.assertIsNone(annotations[0]["file"]) + self.assertIsNone(annotations[0]["line"]) + self.assertEqual(annotations[0]["title"], "roottest-mystery-crash") + + def test_annotate_ctest_failures_end_to_end(self): + with tempfile.TemporaryDirectory() as tmpdir: + temp_build = os.path.join(tmpdir, "build", "Testing", "Temporary") + os.makedirs(temp_build) + with open(os.path.join(temp_build, "LastTestsFailed.log"), "w", encoding="utf-8") as f: + f.write("1:test-sample\n") + with open(os.path.join(temp_build, "LastTest.log"), "w", encoding="utf-8") as f: + f.write(""" +1/1 Testing: test-sample +---------------------------------------------------------- +/dummy/src/roottest/sample.C:10: error: failed assertion +---------------------------------------------------------- +Test Failed. +""") + # Should run without error + build_utils.annotate_ctest_failures(tmpdir) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/test-ci-scripts.yml b/.github/workflows/test-ci-scripts.yml index 71992410bcb0a..71c9f58af0fd2 100644 --- a/.github/workflows/test-ci-scripts.yml +++ b/.github/workflows/test-ci-scripts.yml @@ -15,4 +15,6 @@ jobs: uses: actions/checkout@v6 - name: Run unit tests - run: python3 .github/workflows/root-ci-config/test_null_build_check.py --verbose + run: | + python3 .github/workflows/root-ci-config/test_null_build_check.py --verbose + python3 .github/workflows/root-ci-config/test_annotate_failures.py --verbose