Improve bot finding extraction

online/etl/llm/prompts.py

@@ -1,32 +1,47 @@
 """Prompt templates for the LLM analysis pipeline."""
 
-EXTRACT_BOT_SUGGESTIONS = """You are analyzing a pull request to extract all actionable suggestions made by a code review bot.
+EXTRACT_BOT_SUGGESTIONS = """You are analyzing a pull request to extract concrete code-review findings flagged by a code review bot.
 
 The bot's username is: {bot_username}
 
 Below you will see:
-1. The commits that were under review (the code state the bot saw), including full diffs
-2. The bot's review comments on those commits
+1. The bot's review comments on the commits
+2. The commits that were under review (the code state the bot saw), including full diffs
 
-For each actionable suggestion the bot made, extract:
+The Bot Review Comments section is a numbered list. Each entry is labeled as REVIEW_BODY,
+INLINE_REVIEW_COMMENT, or ISSUE_COMMENT. Process all numbered comments. Top-level inline
+review comments generally contain substantive findings; review bodies and issue comments may
+contain findings or summaries.
+
+Your job is to identify every distinct substantive finding the bot flagged. A finding can be a
+bug, defect, risk, incorrect behavior, missing validation, security concern, performance concern,
+maintainability issue, documentation problem, test gap, or other concrete code-review concern.
+
+Only extract findings grounded in the bot's comments. Do not invent findings solely from the
+commit diff. Use the commit diff only to understand code context for a comment.
+
+For each finding, extract:
 - A unique ID (S1, S2, ...)
-- A description of what was suggested
+- A concise description of the issue or concern the bot flagged
 - The category (bug, style, performance, security, refactor, documentation, other)
 - The file path and line number if available
 - Severity (low, medium, high, critical)
 
-Only include ACTIONABLE suggestions — skip generic praise, summaries, or "looks good" comments.
-Skip bot comments that are purely informational without suggesting any change.
+Skip generic praise, status/progress messages, quota/limit messages, pure summaries without a
+concrete finding, and "looks good" / "no issues found" comments. Also skip follow-up comments
+that merely confirm a fix, acknowledge a clarification, say a parent comment was wrong, too
+broad, pre-existing, already addressed, or out of scope, unless that same comment introduces a
+new distinct code issue.
 
 PR Title: {pr_title}
 PR Author: {pr_author}
 Repository: {repo_name}
 
-=== Commits Under Review (code the bot reviewed) ===
-{commits_under_review}
-
 === Bot Review Comments ===
 {bot_comments}
+
+=== Commits Under Review (code the bot reviewed) ===
+{commits_under_review}
 """
 
 EXTRACT_HUMAN_ACTIONS = """You are analyzing post-review commit diffs to extract every concrete code issue that was fixed or improved AFTER the bot reviewed the PR.

online/etl/llm/schemas.py

@@ -7,8 +7,8 @@
 
 
 class BotSuggestion(BaseModel):
-    issue_id: str = Field(description="Unique identifier for this suggestion (e.g. 'S1', 'S2')")
-    description: str = Field(description="What the bot suggested or flagged")
+    issue_id: str = Field(description="Unique identifier for this finding (e.g. 'S1', 'S2')")
+    description: str = Field(description="The issue or concern the bot flagged")
     category: str = Field(
         description="Category: 'bug', 'style', 'performance', 'security', 'refactor', 'documentation', 'other'"
     )
@@ -18,7 +18,7 @@ class BotSuggestion(BaseModel):
 
 
 class BotSuggestionsResponse(BaseModel):
-    suggestions: list[BotSuggestion] = Field(description="All actionable suggestions made by the bot")
+    suggestions: list[BotSuggestion] = Field(description="All concrete code-review findings flagged by the bot")
 
 
 class HumanAction(BaseModel):

online/etl/pipeline/analyze.py

@@ -5,6 +5,7 @@
 import asyncio
 import json
 import logging
+import re
 
 from openai import BadRequestError
 
@@ -22,6 +23,16 @@
 
 logger = logging.getLogger(__name__)
 
+_HTML_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL)
+_DETAILS_RE = re.compile(r"<details\b[^>]*>.*?</details>", re.DOTALL | re.IGNORECASE)
+_SUB_RE = re.compile(r"<sub\b[^>]*>.*?</sub>", re.DOTALL | re.IGNORECASE)
+_PROMO_FOOTER_HINTS = (
+    "if you found this review helpful",
+    "coderabbit",
+    "review details",
+    "configuration used",
+)
+
 
 def _find_bot_review_commit(
     reviews: list[dict],
@@ -142,13 +153,32 @@ def _format_commits_with_diffs(commits: list[dict], details_by_sha: dict[str, di
     return "\n".join(lines)
 
 
+def _clean_bot_comment_body(body: str) -> str:
+    """Remove hidden/generated Markdown that is not part of the review finding text."""
+    cleaned = _HTML_COMMENT_RE.sub("", body)
+    cleaned = _DETAILS_RE.sub("", cleaned)
+    cleaned = _SUB_RE.sub("", cleaned)
+
+    lines = cleaned.splitlines()
+    for idx, line in enumerate(lines):
+        if line.strip() != "---":
+            continue
+        tail = "\n".join(lines[idx + 1 :]).lower()
+        if any(hint in tail for hint in _PROMO_FOOTER_HINTS):
+            lines = lines[:idx]
+            break
+
+    return "\n".join(lines).strip()
+
+
 def _format_bot_comments(events: list[dict], chatbot_username: str) -> str:
     """Format bot's review/review_comment/issue_comment events with full context.
 
     Skips review_comment replies (in_reply_to_id set) — these are responses
     to other commenters' threads, not original review suggestions.
     """
     lines = []
+    comment_num = 1
     for e in events:
         if not same_github_actor(e.get("actor"), chatbot_username):
             continue
@@ -164,23 +194,29 @@ def _format_bot_comments(events: list[dict], chatbot_username: str) -> str:
 
         if etype == "review":
             state = data.get("state", "")
-            body = data.get("body") or ""
-            lines.append(f"[{ts}] REVIEW ({state}):")
+            body = _clean_bot_comment_body(data.get("body") or "")
+            lines.append(f"COMMENT C{comment_num} [REVIEW_BODY state={state} timestamp={ts}]")
             if body:
-                lines.append(f"  {body}")
+                lines.append(body)
         elif etype in ("review_comment", "issue_comment"):
-            body = data.get("body") or ""
+            body = _clean_bot_comment_body(data.get("body") or "")
             path = data.get("path") or ""
             line = data.get("line") or ""
-            loc = f" ({path}:{line})" if path else ""
             diff_hunk = data.get("diff_hunk") or ""
             resolved = " [RESOLVED]" if data.get("is_resolved") else ""
-            lines.append(f"[{ts}] {etype.upper()}{loc}{resolved}:")
+            if etype == "review_comment":
+                label = "INLINE_REVIEW_COMMENT"
+                location = f" path={path}:{line}" if path else ""
+            else:
+                label = "ISSUE_COMMENT"
+                location = ""
+            lines.append(f"COMMENT C{comment_num} [{label}{location}{resolved} timestamp={ts}]")
             if diff_hunk:
-                lines.append(f"  Code context:\n  ```\n{diff_hunk}\n  ```")
+                lines.append(f"Code context:\n```diff\n{diff_hunk}\n```")
             if body:
-                lines.append(f"  {body}")
+                lines.append(body)
         lines.append("")
+        comment_num += 1
     return "\n".join(lines) if lines else "(no bot comments)"
 
 

online/etl/tests/test_analyze_formatting.py

@@ -0,0 +1,84 @@
+"""Tests for bot comment formatting used by LLM extraction."""
+
+from __future__ import annotations
+
+from pipeline.analyze import _clean_bot_comment_body
+from pipeline.analyze import _format_bot_comments
+
+
+def test_clean_bot_comment_body_removes_hidden_metadata() -> None:
+    body = """<!-- hidden classifier notes -->
+Actual review finding.
+
+<details>
+<summary>Debug trace</summary>
+Internal prompt text
+</details>
+
+<sub>generated metadata</sub>
+"""
+
+    assert _clean_bot_comment_body(body) == "Actual review finding."
+
+
+def test_clean_bot_comment_body_removes_promotional_footer() -> None:
+    body = """Potential null dereference here.
+
+---
+If you found this review helpful, react to this comment.
+"""
+
+    assert _clean_bot_comment_body(body) == "Potential null dereference here."
+
+
+def test_clean_bot_comment_body_preserves_configured_findings_after_separator() -> None:
+    body = """Initial context.
+
+---
+P2: This option is configured incorrectly and can fail at runtime.
+"""
+
+    assert (
+        _clean_bot_comment_body(body)
+        == "Initial context.\n\n---\nP2: This option is configured incorrectly and can fail at runtime."
+    )
+
+
+def test_format_bot_comments_labels_and_numbers_comments() -> None:
+    events = [
+        {
+            "actor": "reviewer[bot]",
+            "event_type": "review_comment",
+            "timestamp": "2026-07-01T12:00:00Z",
+            "data": {
+                "path": "src/app.py",
+                "line": 42,
+                "diff_hunk": "@@ -1 +1 @@\n-old\n+new",
+                "body": "This can throw when value is null.",
+            },
+        },
+        {
+            "actor": "reviewer[bot]",
+            "event_type": "review_comment",
+            "timestamp": "2026-07-01T12:01:00Z",
+            "data": {
+                "in_reply_to_id": 123,
+                "path": "src/app.py",
+                "line": 42,
+                "body": "Fixed now.",
+            },
+        },
+        {
+            "actor": "reviewer[bot]",
+            "event_type": "review",
+            "timestamp": "2026-07-01T12:02:00Z",
+            "data": {"state": "commented", "body": "Summary finding."},
+        },
+    ]
+
+    formatted = _format_bot_comments(events, "reviewer[bot]")
+
+    assert "COMMENT C1 [INLINE_REVIEW_COMMENT path=src/app.py:42 timestamp=2026-07-01T12:00:00Z]" in formatted
+    assert "Code context:\n```diff\n@@ -1 +1 @@\n-old\n+new\n```" in formatted
+    assert "COMMENT C2 [REVIEW_BODY state=commented timestamp=2026-07-01T12:02:00Z]" in formatted
+    assert "Fixed now." not in formatted