Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .github/scripts/pr_automation.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
'ai:failed': 'Review failed',
'ai:skipped': 'Review skipped',
}
AI_ALERT_LABELS = frozenset(('ai:findings', 'ai:failed'))
# Include retired labels so synchronization removes them from existing PRs.
AI_LABELS = frozenset(AI_TITLES)
AI_SUCCESS = frozenset(('ai:no-findings', 'ai:findings'))
PREVIOUS_REVIEW = '\n\n### Previous successful review\n\n'
Expand Down Expand Up @@ -201,21 +203,18 @@ def triage_event() -> None:

def review_label(state: dict, previous: str, sha: str) -> str:
if 'result_sha' in state:
if state['result_sha'] == sha and state.get('outcome') in AI_TITLES:
if state['result_sha'] == sha and state.get('outcome') in AI_ALERT_LABELS:
return state['outcome']
return ''
# Summaries published before review labels used a standalone Commit line.
if not re.search(rf'^Commit {sha[:12]}$', previous, re.MULTILINE):
return ''
if state.get('size_skip_sha') == sha and 'Review skipped' in previous:
return 'ai:skipped'
return ''
if 'The model did not return a review.' in previous or 'could be located' in previous:
return 'ai:failed'
if state.get('reviewed'):
if re.search(r'^- \*\*', previous, re.MULTILINE):
return 'ai:findings'
if 'PR-Agent found no actionable issues' in previous:
return 'ai:no-findings'
if state.get('reviewed') and re.search(r'^- \*\*', previous, re.MULTILINE):
return 'ai:findings'
return ''


Expand Down Expand Up @@ -449,7 +448,8 @@ def publish(number: int, sha: str, command: int, directory: Path) -> None:
github(f'issues/{number}/comments', '-X', 'POST', body=comment)
else:
github(f'issues/comments/{comment_id}', '-X', 'PATCH', body=comment)
replace_family(number, {label['name'] for label in pr['labels']}, AI_LABELS, outcome)
label = outcome if outcome in AI_ALERT_LABELS else ''
replace_family(number, {label['name'] for label in pr['labels']}, AI_LABELS, label)
if outcome == 'ai:failed':
raise SystemExit('PR-Agent did not return a review.')

Expand Down
47 changes: 47 additions & 0 deletions .github/scripts/test_pr_automation.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,53 @@ def github(path: str, *args: str, body: dict | None = None) -> None:
]


@pytest.mark.parametrize(
('outcome', 'label'),
[('ai:no-findings', ''), ('ai:skipped', ''), ('ai:findings', 'ai:findings'), ('ai:failed', 'ai:failed')],
)
def test_review_label_only_restores_alerts(outcome: str, label: str) -> None:
assert bot.review_label({'result_sha': SHA, 'outcome': outcome}, '', SHA) == label


@pytest.mark.parametrize(
('result', 'title'),
[('review', 'No actionable findings'), ('size-skip', 'Review skipped')],
)
def test_quiet_review_updates_comment_and_clears_labels(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, result: str, title: str
) -> None:
(tmp_path / 'request.json').write_text(json.dumps({'files': [], 'skip_reason': 'Diff too large.'}))
(tmp_path / 'result.json').write_text(
json.dumps(
{
'review': {'key_issues_to_review': []},
'coverage': {'files_not_reviewed': [], 'files_partially_reviewed': []},
}
)
)
monkeypatch.setenv('REVIEW_RESULT', result)
monkeypatch.setenv('GITHUB_SERVER_URL', 'https://github.com')
monkeypatch.setenv('GITHUB_REPOSITORY', 'owner/repo')
monkeypatch.setenv('GITHUB_RUN_ID', '918')
labels = [{'name': name} for name in ('ai:findings', 'ai:no-findings', 'area:core')]
monkeypatch.setattr(bot, 'current_pr', lambda *_: dict(PR, labels=labels))
monkeypatch.setattr(bot, 'previous_review', lambda _: (17, {}, ''))
calls = []

def github(path: str, *args: str, body: dict | None = None) -> None:
calls.append((path, args, body))

monkeypatch.setattr(bot, 'github', github)
bot.publish(42, SHA, 0, tmp_path)
path, args, payload = calls[0]
assert (path, args) == ('issues/comments/17', ('-X', 'PATCH'))
assert f'**{title}**' in payload['body']
assert calls[1:] == [
('issues/42/labels/ai%3Afindings', ('-X', 'DELETE'), None),
('issues/42/labels/ai%3Ano-findings', ('-X', 'DELETE'), None),
]


def test_triage_pr_expires_status_for_an_old_commit(monkeypatch: pytest.MonkeyPatch) -> None:
labels = ['area:core', 'needs-info', 'ai:findings']
monkeypatch.setattr(bot, 'get', lambda _: dict(PR, labels=[{'name': label} for label in labels]))
Expand Down
11 changes: 5 additions & 6 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,15 @@ area labels from their changed files.
- `ready-for-review` means a maintainer has checked a PR's scope and validation evidence.

Each item has one triage status. Use GitHub's review requests, approvals, and change requests
to manage the human review. The bot adds one `ai:*` label for the reviewed commit.
to manage the human review. The bot labels findings or failures for the current commit.

- `ai:no-findings` means the review finished without actionable findings.
- `ai:findings` means the reviewer returned findings for a maintainer to verify.
- `ai:failed` means the review did not produce a usable result.
- `ai:skipped` means the bot skipped review and recorded the reason in its summary.

PR Intake clears the AI label when the PR head changes. The summary shows the last attempt,
commit, completion time, and the Actions run. A failed re-review preserves the previous
successful review for reference.
The review summary records every outcome, including reviews with no actionable findings
and skipped reviews with their reason. It shows the last attempt, commit, completion time,
and the Actions run. PR Intake clears the AI label when the PR head changes. A failed re-review
preserves the previous successful review for reference.

### Operating the AI review workflow

Expand Down
Loading