From 2403f7b85c57318378c78e93f847690e6c3c9f85 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 21 Sep 2026 19:25:51 +0000 Subject: [PATCH] ci: add manual bypasses for live OAR checks --- .github/scripts/pr_review.py | 15 ++++++- .github/scripts/review_report.py | 13 ++++++ .github/workflows/oar-integration.yml | 33 +++++++++++++-- .github/workflows/pr-review.yml | 3 +- docs/development/ci.md | 39 ++++++++++++++++-- tests/test_pr_review.py | 58 +++++++++++++++++++++++++++ tests/test_review_report.py | 17 ++++++++ 7 files changed, 170 insertions(+), 8 deletions(-) diff --git a/.github/scripts/pr_review.py b/.github/scripts/pr_review.py index e0fa0789..9883d16a 100644 --- a/.github/scripts/pr_review.py +++ b/.github/scripts/pr_review.py @@ -59,6 +59,13 @@ def resolve_request(github, context): ] if not tasks: return _retirement_request(github, pr) + bypass = "" + if context.get("skip_live"): + bypass = "Repository variable OAR_SKIP_LIVE=true." + elif any( + label["name"].lower() == "skip-oar-live" for label in pr.get("labels", []) + ): + bypass = "PR label skip-oar-live." return { "number": number, "head": pr["head"]["sha"], @@ -67,6 +74,7 @@ def resolve_request(github, context): "description": pr.get("body") or "", "tasks": tasks, "reason": "", + "bypass": bypass, } @@ -80,6 +88,7 @@ def main(): Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8") ), "event_name": os.environ["GITHUB_EVENT_NAME"], + "skip_live": os.environ.get("OAR_SKIP_LIVE", "").lower() == "true", } request = resolve_request(GitHub(), context) if request is None: @@ -96,7 +105,11 @@ def main(): args.output.write_text(json.dumps(request, indent=2) + "\n", encoding="utf-8") outputs = {key: request[key] for key in ("number", "head", "base")} outputs.update( - ready=str(bool(request["tasks"]) and not request["reason"]).lower(), + ready=str( + bool(request["tasks"]) + and not request["reason"] + and not request.get("bypass") + ).lower(), tooling=tooling, ) with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output: diff --git a/.github/scripts/review_report.py b/.github/scripts/review_report.py index 500624d0..a9db0158 100644 --- a/.github/scripts/review_report.py +++ b/.github/scripts/review_report.py @@ -122,6 +122,19 @@ def render_report(*, request, reviews, run_url, run_id, outcome): ) if request.get("tooling"): lines.extend([f"Reviewer and guidelines revision: `{request['tooling']}`", ""]) + if request.get("bypass"): + lines.extend( + [ + "> [!WARNING]", + "> Live OAR review was manually bypassed. No verdict was produced; this is not a passing review.", + f"> {_escape_text(request['bypass'])}", + "", + "Projects not reviewed:", + "", + *[f"- {_escape_text(task['label'])}" for task in request["tasks"]], + ] + ) + return "\n".join(lines) if reviews: lines.extend( [ diff --git a/.github/workflows/oar-integration.yml b/.github/workflows/oar-integration.yml index 3367fe02..3390f598 100644 --- a/.github/workflows/oar-integration.yml +++ b/.github/workflows/oar-integration.yml @@ -2,7 +2,7 @@ name: OAR live integration "on": pull_request: - types: [opened, reopened, synchronize, ready_for_review] + types: [opened, reopened, synchronize, ready_for_review, labeled, unlabeled] paths: - .github/workflows/oar-integration.yml - .github/actions/setup-review-gateway/** @@ -28,16 +28,43 @@ concurrency: cancel-in-progress: true jobs: + bypass: + name: Report manual live OAR bypass + if: >- + vars.OAR_SKIP_LIVE == 'true' || + contains(github.event.pull_request.labels.*.name, 'skip-oar-live') + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: Explain bypass + env: + REPOSITORY_BYPASS: ${{ vars.OAR_SKIP_LIVE == 'true' }} + run: | + if [[ "$REPOSITORY_BYPASS" == 'true' ]]; then + reason='Repository variable OAR_SKIP_LIVE=true.' + else + reason='PR label skip-oar-live.' + fi + echo "::warning::Live OAR integration manually bypassed. $reason" + { + echo '## Live OAR integration manually bypassed' + echo + echo "$reason" + echo 'No live execution was verified. Offline functional, package, and runtime checks remain enabled.' + } >> "$GITHUB_STEP_SUMMARY" + integration: name: Run installed OAR through OpenShell # Candidate OAR code runs on the host: restrict secrets to trusted branches. if: >- - (github.event_name == 'pull_request' && + vars.OAR_SKIP_LIVE != 'true' && + !contains(github.event.pull_request.labels.*.name, 'skip-oar-live') && + ((github.event_name == 'pull_request' && !github.event.pull_request.draft && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') || (github.event_name != 'pull_request' && - github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) + github.ref == format('refs/heads/{0}', github.event.repository.default_branch))) runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index 9521b09b..90e510e6 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -3,7 +3,7 @@ name: New project review "on": # Trusted workflow/tooling only. PR contents are uploaded as data, never run here. pull_request_target: - types: [opened, reopened, synchronize, ready_for_review, converted_to_draft] + types: [opened, reopened, synchronize, ready_for_review, converted_to_draft, labeled, unlabeled] permissions: {} @@ -44,6 +44,7 @@ jobs: id: request env: GH_TOKEN: ${{ github.token }} + OAR_SKIP_LIVE: ${{ vars.OAR_SKIP_LIVE }} run: uv run --project tooling/projects/tools/openshell-agent-runner --locked --no-dev python tooling/.github/scripts/pr_review.py --tooling tooling --output request/request.json - name: Upload request if: always() && steps.request.outputs.number != '' diff --git a/docs/development/ci.md b/docs/development/ci.md index 32520f59..a0d7f152 100644 --- a/docs/development/ci.md +++ b/docs/development/ci.md @@ -82,9 +82,9 @@ findings are advisory; native checks remain separate merge gates. ## Execution and trust -`New project review` runs on PR opening, reopening, new commits, readiness, and -conversion back to draft. When a new project is removed from the PR or the PR -returns to draft, the workflow removes its now-stale bot report. +`New project review` runs on PR opening, reopening, new commits, readiness, +label changes, and conversion back to draft. When a new project is removed from +the PR or the PR returns to draft, the workflow removes its now-stale bot report. Draft, fork, and Dependabot PRs skip live review. There is no manual fork authorization or waiting for other workflows. @@ -134,6 +134,39 @@ not the sandbox or comment reporter. No persistent gateway is required. Dependency-license checks are already part of the repository's CI. They neither use OAR nor coordinate with this reviewer; see [Dependency License Checks](dependency-licenses.md). +## Bypass live checks during an external outage + +Maintainers can explicitly bypass new-project reviews and live OAR integration +when inference or another external dependency is unavailable: + +- For one PR, apply the `skip-oar-live` label. Create it first if needed. + Adding or removing it triggers the applicable workflows; remove it to resume + live checks. It persists across commits until removed. +- For a repository-wide outage, set the Actions **variable** `OAR_SKIP_LIVE` to + `true` in **Settings → Secrets and variables → Actions → Variables**. Delete + the variable or set it to `false` after recovery. Variable changes do not + trigger workflows: rerun **all jobs** in affected runs to apply the new value. + +These controls require the updated workflows on the default branch (and the +updated live integration workflow in the tested PR). Rerunning an older workflow +revision does not add bypass support; trigger a new run using the updated revision. +PR label controls use GitHub's label permissions; restrict label management to +the people allowed to waive live checks. The integration workflow uses labels +from its event payload, so use the new label-change run rather than rerunning +an older event to apply label changes. + +The PR report and integration summary explicitly state that execution was +manually bypassed, with no passing verdict. Offline OAR functional, package, and +runtime checks, and all other repository checks, remain enabled. Provider errors +still fail live runs unless someone explicitly enables a bypass. These controls +do not change branch protection or dismiss other failures. + +To diagnose a failure, download `pr-review-results` and inspect `review-*.log`. +A provider HTTP 503 after sandbox setup and input uploads indicates an upstream +inference failure, not a review finding. A successful gateway setup only confirms +configuration; verify the configured model with an authenticated Chat Completions +request or rerun live integration after recovery. Do not print credentials. + ## Validation and evolution CI scripts are Python; requests use PyYAML from OAR's locked environment and diff --git a/tests/test_pr_review.py b/tests/test_pr_review.py index 45ecabdc..753424ec 100644 --- a/tests/test_pr_review.py +++ b/tests/test_pr_review.py @@ -13,6 +13,8 @@ from pathlib import Path from unittest.mock import patch +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / ".github/scripts")) from pr_review import main, resolve_request @@ -242,5 +244,61 @@ def test_cli_marks_report_retirement_as_not_ready_for_inference(self): self.assertIn("ready=false\n", output.read_text()) +@pytest.mark.parametrize( + ("variable", "labels", "ready", "bypass"), + [ + ("", [], True, ""), + ("false", ["documentation"], True, ""), + ("true", [], False, "Repository variable OAR_SKIP_LIVE=true."), + ("TRUE", [], False, "Repository variable OAR_SKIP_LIVE=true."), + ("", ["skip-oar-live"], False, "PR label skip-oar-live."), + ("", ["Skip-OAR-Live"], False, "PR label skip-oar-live."), + ("false", ["skip-oar-live"], False, "PR label skip-oar-live."), + ], +) +def test_cli_live_bypass_preserves_scope_and_disables_inference( + tmp_path, monkeypatch, variable, labels, ready, bypass +): + github = MockGitHub() + event = tmp_path / "event.json" + # The latest API labels, not a stale event's labels, control the PR review. + event.write_text(json.dumps({"pull_request": github.pr})) + github.pr["labels"] = [{"name": label} for label in labels] + request_file = tmp_path / "request.json" + output = tmp_path / "outputs" + for key, value in { + "GITHUB_EVENT_PATH": str(event), + "GITHUB_EVENT_NAME": "pull_request_target", + "GITHUB_OUTPUT": str(output), + "OAR_SKIP_LIVE": variable, + }.items(): + monkeypatch.setenv(key, value) + monkeypatch.setattr("pr_review.GitHub", lambda: github) + monkeypatch.setattr("sys.argv", ["pr_review.py", "--output", str(request_file)]) + with patch( + "pr_review.subprocess.run", + return_value=subprocess.CompletedProcess([], 0, "c" * 40 + "\n"), + ): + main() + + saved = json.loads(request_file.read_text()) + assert saved["bypass"] == bypass + assert saved["head"] == HEAD + assert saved["tasks"][0]["input"] == "projects/tools/new" + assert f"ready={str(ready).lower()}\n" in output.read_text() + + +def test_removing_bypass_label_resumes_review_despite_stale_payload(): + github = MockGitHub() + payload = copy.deepcopy(github.pr) + payload["labels"] = [{"name": "skip-oar-live"}] + request = resolve_request( + github, + {"event_name": "pull_request_target", "payload": {"pull_request": payload}}, + ) + assert request["tasks"] + assert not request["bypass"] + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_review_report.py b/tests/test_review_report.py index ac4ce6e0..07531d43 100644 --- a/tests/test_review_report.py +++ b/tests/test_review_report.py @@ -67,6 +67,23 @@ def report_options(): } +def test_manual_bypass_report_never_presents_stale_results_as_passing(): + options = report_options() + options["request"].update( + bypass="PR label skip-oar-live.", + tasks=[{"label": "projects/research/new-spike"}], + ) + options["outcome"] = "skipped" + body = render_report(**options) + assert REVIEW_MARKER in body + assert "manually bypassed" in body + assert "No verdict was produced" in body + assert "PR label skip-oar-live" in body + assert "projects/research/new-spike" in body + assert "✅ Pass" not in body + assert "No result or execution status" not in body + + class MockGitHub: def __init__(self): self.pr = {"number": 7, "state": "open", "head": {"sha": HEAD}}