From 29caa247b3fff8770118955727671628238b00a9 Mon Sep 17 00:00:00 2001 From: Adam Bowker Date: Mon, 20 Jul 2026 14:03:11 -0400 Subject: [PATCH 1/2] feat(stamphog): trust clean ReviewHog signals Generated-By: PostHog Code Task-Id: fa3ff167-712b-41c4-aa27-aa8ec1700f4e --- tools/pr-approval-agent/github.py | 33 ++++++++++++- tools/pr-approval-agent/reviewer.py | 12 ++++- tools/pr-approval-agent/test_github.py | 64 +++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/tools/pr-approval-agent/github.py b/tools/pr-approval-agent/github.py index 2a8adeb199..4bb87ededb 100644 --- a/tools/pr-approval-agent/github.py +++ b/tools/pr-approval-agent/github.py @@ -6,8 +6,9 @@ """ import json +import re import subprocess -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path @@ -29,6 +30,7 @@ class PRData: reviews: list[dict] review_comments: list[dict] check_runs: list[dict] + pr_reactions: list[dict] = field(default_factory=list) @property def file_paths(self) -> list[str]: @@ -52,6 +54,8 @@ def has_new_files(self) -> bool: _TRUSTED_ASSOCIATIONS = {"MEMBER", "OWNER", "COLLABORATOR"} +_REVIEWHOG_BOT_LOGIN = "posthog[bot]" +_REVIEWHOG_STATUS_RE = re.compile(r"") def _normalize_reviews_for_prompt(reviews_raw: list[dict], head_sha: str) -> list[dict]: @@ -85,6 +89,26 @@ def _normalize_reviews_for_prompt(reviews_raw: list[dict], head_sha: str) -> lis return normalized_reviews +def _reviewhog_clean_reactions(comments_raw: list[dict]) -> list[dict]: + for comment in reversed(comments_raw): + user = comment.get("user") or {} + if user.get("login", "").lower() != _REVIEWHOG_BOT_LOGIN or user.get("type") != "Bot": + continue + body = comment.get("body") or "" + if "Found no issues worth raising, so no review was posted." not in body: + continue + if _REVIEWHOG_STATUS_RE.search(body) is None: + continue + return [ + { + "user": "reviewhog[bot]", + "emoji": "👍", + "created_at": comment.get("updated_at") or comment.get("created_at"), + } + ] + return [] + + def _gh_api(endpoint: str, *, paginate: bool = False) -> dict | list: cmd = ["gh", "api", endpoint] if paginate: @@ -258,6 +282,12 @@ def fetch_pr(pr_number: int, repo: str, repo_root: Path | None = None) -> PRData base_sha = pr["base"]["sha"] head_sha = pr["head"]["sha"] check_runs_resp = _gh_api(f"repos/{repo}/commits/{head_sha}/check-runs") + reviewhog_reactions: list[dict] = [] + try: + discussion_raw = _gh_api(f"repos/{repo}/issues/{pr_number}/comments", paginate=True) + reviewhog_reactions = _reviewhog_clean_reactions(discussion_raw) + except Exception as exc: + print(f"warning: discussion fetch failed ({exc}); continuing without ReviewHog assurance") git_root = repo_root or Path.cwd() ensure_commits(pr_number, head_sha, git_root) @@ -280,6 +310,7 @@ def fetch_pr(pr_number: int, repo: str, repo_root: Path | None = None) -> PRData reviews=_normalize_reviews_for_prompt(reviews_raw, head_sha), review_comments=review_comments, check_runs=check_runs_resp.get("check_runs", []), + pr_reactions=reviewhog_reactions, ) diff --git a/tools/pr-approval-agent/reviewer.py b/tools/pr-approval-agent/reviewer.py index ea935011a6..f7a8ebdcd7 100644 --- a/tools/pr-approval-agent/reviewer.py +++ b/tools/pr-approval-agent/reviewer.py @@ -174,6 +174,9 @@ def _validate_verdict(result: dict) -> dict: reviews as a concern and ESCALATE unless there's a strong, specific justification to APPROVE. - Bot comments with valid concerns that were ignored → ESCALATE + - ReviewHog's authenticated clean status appears as 👍 @reviewhog[bot]. Treat + it as the same mild positive signal as a clean Greptile or Hex reaction: + useful corroboration, but never sufficient by itself to approve. Tools: You have Read, Grep, and Glob (restricted to the repo directory). All PR metadata (comments, ownership) is in the prompt — do NOT fetch @@ -392,6 +395,10 @@ def _build_review_prompt(self, pr: PRData, cl: dict, gate_context: dict, diff_pa lines.append(f" - @{safe_user}{reply}{status} on {safe_path}: {safe_body}") review_comments = "\n".join(lines) + pr_reactions = "\n".join( + f" - {r['emoji']} @{_sanitize_untrusted(r['user'], max_len=50)}" for r in pr.pr_reactions + ) + ownership = self._format_ownership(cl) gate_lines = [] @@ -419,7 +426,7 @@ def _build_review_prompt(self, pr: PRData, cl: dict, gate_context: dict, diff_pa Size: {pr.lines_total} lines ({pr.lines_added}+/{pr.lines_deleted}-), {len(pr.files)} files Scope: {cl["breadth"]} Commit type: {cl.get("commit_type") or "unknown"} - Reviews: {len(pr.reviews)} top-level, {len(pr.review_comments)} inline + Reviews: {len(pr.reviews)} top-level, {len(pr.review_comments)} inline, {len(pr.pr_reactions)} trusted signals {ownership} @@ -443,6 +450,9 @@ def _build_review_prompt(self, pr: PRData, cl: dict, gate_context: dict, diff_pa Inline comments: {review_comments} + + Reactions on the PR: + {pr_reactions} --- END UNTRUSTED CONTENT --- """) diff --git a/tools/pr-approval-agent/test_github.py b/tools/pr-approval-agent/test_github.py index 87aaa5a461..8f4cf940d2 100644 --- a/tools/pr-approval-agent/test_github.py +++ b/tools/pr-approval-agent/test_github.py @@ -2,7 +2,7 @@ import pytest -from github import _normalize_reviews_for_prompt +from github import _normalize_reviews_for_prompt, _reviewhog_clean_reactions def test_normalize_reviews_marks_current_head_and_preserves_stale_reviews() -> None: @@ -75,3 +75,65 @@ def test_normalize_reviews_filters_by_trust_source( ) assert len(normalized) == expected_count + + +def test_reviewhog_clean_status_is_normalized_as_a_trusted_positive() -> None: + reactions = _reviewhog_clean_reactions( + [ + { + "user": {"login": "posthog[bot]", "type": "Bot"}, + "body": ( + "Found no issues worth raising, so no review was posted.\n\n" + "" + ), + "created_at": "2026-07-20T17:04:26Z", + "updated_at": "2026-07-20T17:05:26Z", + } + ] + ) + + assert reactions == [ + { + "user": "reviewhog[bot]", + "emoji": "👍", + "created_at": "2026-07-20T17:05:26Z", + } + ] + + +def _clean_reviewhog_body() -> str: + return ( + "Found no issues worth raising, so no review was posted.\n\n" + "" + ) + + +@pytest.mark.parametrize( + "login,user_type,body", + [ + pytest.param("posthog[bot]", "User", _clean_reviewhog_body(), id="not-app-bot"), + pytest.param("someone[bot]", "Bot", _clean_reviewhog_body(), id="wrong-bot"), + pytest.param( + "posthog[bot]", "Bot", "Found no issues worth raising, so no review was posted.", id="missing-marker" + ), + pytest.param( + "posthog[bot]", + "Bot", + "Found 1 should fix.\n", + id="not-clean", + ), + ], +) +def test_reviewhog_clean_status_rejects_untrusted_or_non_clean_comments( + login: str, user_type: str, body: str +) -> None: + reactions = _reviewhog_clean_reactions( + [ + { + "user": {"login": login, "type": user_type}, + "body": body, + } + ] + ) + + assert reactions == [] From d190be88fefc1d2b8c36ef993ad9f6c2c4f5ff70 Mon Sep 17 00:00:00 2001 From: Adam Bowker Date: Mon, 20 Jul 2026 14:20:33 -0400 Subject: [PATCH 2/2] fix(stamphog): consume trusted PR reactions Generated-By: PostHog Code Task-Id: fa3ff167-712b-41c4-aa27-aa8ec1700f4e --- tools/pr-approval-agent/github.py | 47 ++++++++++----------- tools/pr-approval-agent/reviewer.py | 6 +-- tools/pr-approval-agent/test_github.py | 56 +++++++++----------------- 3 files changed, 45 insertions(+), 64 deletions(-) diff --git a/tools/pr-approval-agent/github.py b/tools/pr-approval-agent/github.py index 4bb87ededb..9d7f403dc9 100644 --- a/tools/pr-approval-agent/github.py +++ b/tools/pr-approval-agent/github.py @@ -6,7 +6,6 @@ """ import json -import re import subprocess from dataclasses import dataclass, field from pathlib import Path @@ -54,8 +53,14 @@ def has_new_files(self) -> bool: _TRUSTED_ASSOCIATIONS = {"MEMBER", "OWNER", "COLLABORATOR"} -_REVIEWHOG_BOT_LOGIN = "posthog[bot]" -_REVIEWHOG_STATUS_RE = re.compile(r"") +TRUSTED_REACTOR_BOTS = { + "chatgpt-codex-connector[bot]", + "copilot-pull-request-reviewer[bot]", + "greptile-apps[bot]", + "hex-security-app[bot]", + "posthog[bot]", + "veria-ai[bot]", +} def _normalize_reviews_for_prompt(reviews_raw: list[dict], head_sha: str) -> list[dict]: @@ -89,24 +94,20 @@ def _normalize_reviews_for_prompt(reviews_raw: list[dict], head_sha: str) -> lis return normalized_reviews -def _reviewhog_clean_reactions(comments_raw: list[dict]) -> list[dict]: - for comment in reversed(comments_raw): - user = comment.get("user") or {} - if user.get("login", "").lower() != _REVIEWHOG_BOT_LOGIN or user.get("type") != "Bot": - continue - body = comment.get("body") or "" - if "Found no issues worth raising, so no review was posted." not in body: +def _normalize_pr_reactions(reactions_raw: list[dict], author: str) -> list[dict]: + normalized = [] + for reaction in reactions_raw: + login = (reaction.get("user") or {}).get("login", "") + if login == author or login.lower() not in TRUSTED_REACTOR_BOTS: continue - if _REVIEWHOG_STATUS_RE.search(body) is None: - continue - return [ + normalized.append( { - "user": "reviewhog[bot]", - "emoji": "👍", - "created_at": comment.get("updated_at") or comment.get("created_at"), + "user": login, + "emoji": {"+1": "👍", "-1": "👎", "eyes": "👀"}.get(reaction.get("content"), reaction.get("content")), + "created_at": reaction.get("created_at"), } - ] - return [] + ) + return normalized def _gh_api(endpoint: str, *, paginate: bool = False) -> dict | list: @@ -282,12 +283,12 @@ def fetch_pr(pr_number: int, repo: str, repo_root: Path | None = None) -> PRData base_sha = pr["base"]["sha"] head_sha = pr["head"]["sha"] check_runs_resp = _gh_api(f"repos/{repo}/commits/{head_sha}/check-runs") - reviewhog_reactions: list[dict] = [] + pr_reactions: list[dict] = [] try: - discussion_raw = _gh_api(f"repos/{repo}/issues/{pr_number}/comments", paginate=True) - reviewhog_reactions = _reviewhog_clean_reactions(discussion_raw) + reactions_raw = _gh_api(f"repos/{repo}/issues/{pr_number}/reactions", paginate=True) + pr_reactions = _normalize_pr_reactions(reactions_raw, pr["user"]["login"]) except Exception as exc: - print(f"warning: discussion fetch failed ({exc}); continuing without ReviewHog assurance") + print(f"warning: reaction fetch failed ({exc}); continuing without reaction context") git_root = repo_root or Path.cwd() ensure_commits(pr_number, head_sha, git_root) @@ -310,7 +311,7 @@ def fetch_pr(pr_number: int, repo: str, repo_root: Path | None = None) -> PRData reviews=_normalize_reviews_for_prompt(reviews_raw, head_sha), review_comments=review_comments, check_runs=check_runs_resp.get("check_runs", []), - pr_reactions=reviewhog_reactions, + pr_reactions=pr_reactions, ) diff --git a/tools/pr-approval-agent/reviewer.py b/tools/pr-approval-agent/reviewer.py index f7a8ebdcd7..814869da95 100644 --- a/tools/pr-approval-agent/reviewer.py +++ b/tools/pr-approval-agent/reviewer.py @@ -174,9 +174,9 @@ def _validate_verdict(result: dict) -> dict: reviews as a concern and ESCALATE unless there's a strong, specific justification to APPROVE. - Bot comments with valid concerns that were ignored → ESCALATE - - ReviewHog's authenticated clean status appears as 👍 @reviewhog[bot]. Treat - it as the same mild positive signal as a clean Greptile or Hex reaction: - useful corroboration, but never sufficient by itself to approve. + - Trusted reviewer reactions are included. A 👍 is mild positive evidence and + a 👎 is mild negative evidence; neither is sufficient by itself. An 👀 means + a review is still in flight, so do not approve until it finishes. Tools: You have Read, Grep, and Glob (restricted to the repo directory). All PR metadata (comments, ownership) is in the prompt — do NOT fetch diff --git a/tools/pr-approval-agent/test_github.py b/tools/pr-approval-agent/test_github.py index 8f4cf940d2..8068a753e6 100644 --- a/tools/pr-approval-agent/test_github.py +++ b/tools/pr-approval-agent/test_github.py @@ -2,7 +2,7 @@ import pytest -from github import _normalize_reviews_for_prompt, _reviewhog_clean_reactions +from github import _normalize_pr_reactions, _normalize_reviews_for_prompt def test_normalize_reviews_marks_current_head_and_preserves_stale_reviews() -> None: @@ -77,63 +77,43 @@ def test_normalize_reviews_filters_by_trust_source( assert len(normalized) == expected_count -def test_reviewhog_clean_status_is_normalized_as_a_trusted_positive() -> None: - reactions = _reviewhog_clean_reactions( +def test_normalize_pr_reactions_keeps_trusted_reviewer_bots() -> None: + reactions = _normalize_pr_reactions( [ { - "user": {"login": "posthog[bot]", "type": "Bot"}, - "body": ( - "Found no issues worth raising, so no review was posted.\n\n" - "" - ), + "user": {"login": "posthog[bot]"}, + "content": "+1", "created_at": "2026-07-20T17:04:26Z", - "updated_at": "2026-07-20T17:05:26Z", } - ] + ], + "author", ) assert reactions == [ { - "user": "reviewhog[bot]", + "user": "posthog[bot]", "emoji": "👍", - "created_at": "2026-07-20T17:05:26Z", + "created_at": "2026-07-20T17:04:26Z", } ] -def _clean_reviewhog_body() -> str: - return ( - "Found no issues worth raising, so no review was posted.\n\n" - "" - ) - - @pytest.mark.parametrize( - "login,user_type,body", + "login,author", [ - pytest.param("posthog[bot]", "User", _clean_reviewhog_body(), id="not-app-bot"), - pytest.param("someone[bot]", "Bot", _clean_reviewhog_body(), id="wrong-bot"), - pytest.param( - "posthog[bot]", "Bot", "Found no issues worth raising, so no review was posted.", id="missing-marker" - ), - pytest.param( - "posthog[bot]", - "Bot", - "Found 1 should fix.\n", - id="not-clean", - ), + pytest.param("someone[bot]", "author", id="untrusted-bot"), + pytest.param("posthog[bot]", "posthog[bot]", id="author-self-reaction"), ], ) -def test_reviewhog_clean_status_rejects_untrusted_or_non_clean_comments( - login: str, user_type: str, body: str -) -> None: - reactions = _reviewhog_clean_reactions( +def test_normalize_pr_reactions_rejects_untrusted_or_author_reactions(login: str, author: str) -> None: + reactions = _normalize_pr_reactions( [ { - "user": {"login": login, "type": user_type}, - "body": body, + "user": {"login": login}, + "content": "+1", } - ] + ], + author, ) assert reactions == []