From 3136f54b3080fef387e408e4f33fccb4eb3ea35e Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Thu, 10 Sep 2026 15:17:03 -0400 Subject: [PATCH 1/2] ci: make paper qualification fail closed --- .github/workflows/paper.yml | 32 +++++++++------ tests/unit/test_workflow_policy.py | 62 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/.github/workflows/paper.yml b/.github/workflows/paper.yml index 47bd117..46da5a7 100644 --- a/.github/workflows/paper.yml +++ b/.github/workflows/paper.yml @@ -243,23 +243,33 @@ jobs: - name: Require every provider qualification stage to pass if: ${{ always() }} env: + ALPACA_EXERCISE: ${{ steps.alpaca-exercise.outcome }} + ALPACA_RESTART: ${{ steps.alpaca-restart.outcome }} EXTENDED_PROVIDER: ${{ inputs.extended-provider }} + FEED_EVIDENCE: ${{ steps.feed-evidence.outcome }} + FEED_EVIDENCE_SCAN: ${{ steps.feed-evidence-scan.outcome }} + IB_EXERCISE: ${{ steps.ib-exercise.outcome }} + IB_RESTART: ${{ steps.ib-restart.outcome }} + OKX_EXTERNAL: ${{ steps.okx-external.outcome }} + PAPER_EVIDENCE: ${{ steps.evidence.outcome }} + PAPER_EVIDENCE_SCAN: ${{ steps.evidence-scan.outcome }} + PROVIDER_SOAKS: ${{ steps.provider-soaks.outcome }} run: | - test "${{ steps.alpaca-exercise.outcome }}" = success && - test "${{ steps.alpaca-restart.outcome }}" = success && - test "${{ steps.ib-exercise.outcome }}" = success && - test "${{ steps.ib-restart.outcome }}" = success && - test "${{ steps.okx-external.outcome }}" = success && - test "${{ steps.evidence-scan.outcome }}" = success && - test "${{ steps.feed-evidence-scan.outcome }}" = success + test "$ALPACA_EXERCISE" = success + test "$ALPACA_RESTART" = success + test "$IB_EXERCISE" = success + test "$IB_RESTART" = success + test "$OKX_EXTERNAL" = success + test "$PAPER_EVIDENCE_SCAN" = success + test "$FEED_EVIDENCE_SCAN" = success if test "$EXTENDED_PROVIDER" != none; then - test "${{ steps.provider-soaks.outcome }}" = success + test "$PROVIDER_SOAKS" = success fi if test "$EXTENDED_PROVIDER" = all; then - test "${{ steps.evidence.outcome }}" = success - test "${{ steps.feed-evidence.outcome }}" = success + test "$PAPER_EVIDENCE" = success + test "$FEED_EVIDENCE" = success elif test "$EXTENDED_PROVIDER" = okx; then - test "${{ steps.feed-evidence.outcome }}" = success + test "$FEED_EVIDENCE" = success fi archive: diff --git a/tests/unit/test_workflow_policy.py b/tests/unit/test_workflow_policy.py index 25ba7cc..eeafcdd 100644 --- a/tests/unit/test_workflow_policy.py +++ b/tests/unit/test_workflow_policy.py @@ -1,5 +1,7 @@ from __future__ import annotations +import os +import subprocess from copy import deepcopy import pytest @@ -38,6 +40,66 @@ def test_paper_soak_requires_every_short_provider_check() -> None: assert any("ib-exercise" in failure for failure in paper_soak_failures(seeded_job)) +def test_paper_gate_fails_for_each_required_provider_outcome() -> None: + paper = load_workflow(WORKFLOW_ROOT / "paper.yml") + gate = next( + step + for step in paper["jobs"]["paper"]["steps"] + if step.get("name") == "Require every provider qualification stage to pass" + ) + outcomes = { + "ALPACA_EXERCISE": "success", + "ALPACA_RESTART": "success", + "FEED_EVIDENCE": "skipped", + "FEED_EVIDENCE_SCAN": "success", + "IB_EXERCISE": "success", + "IB_RESTART": "success", + "OKX_EXTERNAL": "success", + "PAPER_EVIDENCE": "skipped", + "PAPER_EVIDENCE_SCAN": "success", + "PROVIDER_SOAKS": "skipped", + "EXTENDED_PROVIDER": "none", + } + + def run_gate(env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", "-euo", "pipefail", "-c", gate["run"]], + check=False, + capture_output=True, + env={**os.environ, **env}, + text=True, + ) + + assert run_gate(outcomes).returncode == 0 + for variable in ( + "ALPACA_EXERCISE", + "ALPACA_RESTART", + "IB_EXERCISE", + "IB_RESTART", + "OKX_EXTERNAL", + "PAPER_EVIDENCE_SCAN", + "FEED_EVIDENCE_SCAN", + ): + seeded = {**outcomes, variable: "failure"} + assert run_gate(seeded).returncode != 0, variable + + assert ( + run_gate({**outcomes, "EXTENDED_PROVIDER": "ib", "PROVIDER_SOAKS": "failure"}).returncode + != 0 + ) + all_outcomes = { + **outcomes, + "EXTENDED_PROVIDER": "all", + "FEED_EVIDENCE": "success", + "PAPER_EVIDENCE": "success", + "PROVIDER_SOAKS": "success", + } + assert run_gate(all_outcomes).returncode == 0 + for variable in ("FEED_EVIDENCE", "PAPER_EVIDENCE", "PROVIDER_SOAKS"): + seeded = {**all_outcomes, variable: "failure"} + assert run_gate(seeded).returncode != 0, variable + + def test_paper_qualification_uses_a_clean_explicit_runtime() -> None: paper = load_workflow(WORKFLOW_ROOT / "paper.yml") paper_job = paper["jobs"]["paper"] From 5fa96be2753c727843c7e68edae447241fcd8e1f Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Thu, 10 Sep 2026 15:22:46 -0400 Subject: [PATCH 2/2] test: make paper gate regression portable --- .github/workflows/paper.yml | 31 ++++---- scripts/qualification/check_paper_outcomes.py | 54 +++++++++++++ tests/unit/test_workflow_policy.py | 75 +++++++------------ 3 files changed, 95 insertions(+), 65 deletions(-) create mode 100644 scripts/qualification/check_paper_outcomes.py diff --git a/.github/workflows/paper.yml b/.github/workflows/paper.yml index 46da5a7..531da0b 100644 --- a/.github/workflows/paper.yml +++ b/.github/workflows/paper.yml @@ -254,23 +254,20 @@ jobs: PAPER_EVIDENCE: ${{ steps.evidence.outcome }} PAPER_EVIDENCE_SCAN: ${{ steps.evidence-scan.outcome }} PROVIDER_SOAKS: ${{ steps.provider-soaks.outcome }} - run: | - test "$ALPACA_EXERCISE" = success - test "$ALPACA_RESTART" = success - test "$IB_EXERCISE" = success - test "$IB_RESTART" = success - test "$OKX_EXTERNAL" = success - test "$PAPER_EVIDENCE_SCAN" = success - test "$FEED_EVIDENCE_SCAN" = success - if test "$EXTENDED_PROVIDER" != none; then - test "$PROVIDER_SOAKS" = success - fi - if test "$EXTENDED_PROVIDER" = all; then - test "$PAPER_EVIDENCE" = success - test "$FEED_EVIDENCE" = success - elif test "$EXTENDED_PROVIDER" = okx; then - test "$FEED_EVIDENCE" = success - fi + run: >- + "${{ runner.temp }}/paper-venv/bin/python" + scripts/qualification/check_paper_outcomes.py + --extended-provider "$EXTENDED_PROVIDER" + --alpaca-exercise "$ALPACA_EXERCISE" + --alpaca-restart "$ALPACA_RESTART" + --ib-exercise "$IB_EXERCISE" + --ib-restart "$IB_RESTART" + --okx-external "$OKX_EXTERNAL" + --paper-evidence-scan "$PAPER_EVIDENCE_SCAN" + --feed-evidence-scan "$FEED_EVIDENCE_SCAN" + --provider-soaks "$PROVIDER_SOAKS" + --paper-evidence "$PAPER_EVIDENCE" + --feed-evidence "$FEED_EVIDENCE" archive: name: Retain provider evidence diff --git a/scripts/qualification/check_paper_outcomes.py b/scripts/qualification/check_paper_outcomes.py new file mode 100644 index 0000000..b6e74c9 --- /dev/null +++ b/scripts/qualification/check_paper_outcomes.py @@ -0,0 +1,54 @@ +"""Fail closed unless every required paper qualification outcome succeeded.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence + +SHORT_OUTCOMES = ( + "alpaca-exercise", + "alpaca-restart", + "ib-exercise", + "ib-restart", + "okx-external", + "paper-evidence-scan", + "feed-evidence-scan", +) + + +def outcome_failures(outcomes: dict[str, str], extended_provider: str) -> list[str]: + """Return required qualification stages whose outcomes are not successful.""" + required = list(SHORT_OUTCOMES) + if extended_provider != "none": + required.append("provider-soaks") + if extended_provider == "all": + required.extend(("paper-evidence", "feed-evidence")) + elif extended_provider == "okx": + required.append("feed-evidence") + return [name for name in required if outcomes.get(name) != "success"] + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--extended-provider", + choices=("none", "alpaca", "ib", "okx", "all"), + required=True, + ) + for name in (*SHORT_OUTCOMES, "provider-soaks", "paper-evidence", "feed-evidence"): + parser.add_argument(f"--{name}", required=True) + args = parser.parse_args(argv) + outcomes = { + name: getattr(args, name.replace("-", "_")) + for name in (*SHORT_OUTCOMES, "provider-soaks", "paper-evidence", "feed-evidence") + } + failures = outcome_failures(outcomes, args.extended_provider) + if failures: + print("paper qualification failed: " + ", ".join(failures)) + return 1 + print("paper qualification: PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/test_workflow_policy.py b/tests/unit/test_workflow_policy.py index eeafcdd..6522032 100644 --- a/tests/unit/test_workflow_policy.py +++ b/tests/unit/test_workflow_policy.py @@ -1,11 +1,10 @@ from __future__ import annotations -import os -import subprocess from copy import deepcopy import pytest +from scripts.qualification.check_paper_outcomes import outcome_failures from scripts.qualification.check_workflows import ( WORKFLOW_ROOT, action_pin_failures, @@ -41,63 +40,43 @@ def test_paper_soak_requires_every_short_provider_check() -> None: def test_paper_gate_fails_for_each_required_provider_outcome() -> None: - paper = load_workflow(WORKFLOW_ROOT / "paper.yml") - gate = next( - step - for step in paper["jobs"]["paper"]["steps"] - if step.get("name") == "Require every provider qualification stage to pass" - ) outcomes = { - "ALPACA_EXERCISE": "success", - "ALPACA_RESTART": "success", - "FEED_EVIDENCE": "skipped", - "FEED_EVIDENCE_SCAN": "success", - "IB_EXERCISE": "success", - "IB_RESTART": "success", - "OKX_EXTERNAL": "success", - "PAPER_EVIDENCE": "skipped", - "PAPER_EVIDENCE_SCAN": "success", - "PROVIDER_SOAKS": "skipped", - "EXTENDED_PROVIDER": "none", + "alpaca-exercise": "success", + "alpaca-restart": "success", + "feed-evidence": "skipped", + "feed-evidence-scan": "success", + "ib-exercise": "success", + "ib-restart": "success", + "okx-external": "success", + "paper-evidence": "skipped", + "paper-evidence-scan": "success", + "provider-soaks": "skipped", } - def run_gate(env: dict[str, str]) -> subprocess.CompletedProcess[str]: - return subprocess.run( - ["bash", "-euo", "pipefail", "-c", gate["run"]], - check=False, - capture_output=True, - env={**os.environ, **env}, - text=True, - ) - - assert run_gate(outcomes).returncode == 0 + assert outcome_failures(outcomes, "none") == [] for variable in ( - "ALPACA_EXERCISE", - "ALPACA_RESTART", - "IB_EXERCISE", - "IB_RESTART", - "OKX_EXTERNAL", - "PAPER_EVIDENCE_SCAN", - "FEED_EVIDENCE_SCAN", + "alpaca-exercise", + "alpaca-restart", + "ib-exercise", + "ib-restart", + "okx-external", + "paper-evidence-scan", + "feed-evidence-scan", ): seeded = {**outcomes, variable: "failure"} - assert run_gate(seeded).returncode != 0, variable + assert outcome_failures(seeded, "none") == [variable] - assert ( - run_gate({**outcomes, "EXTENDED_PROVIDER": "ib", "PROVIDER_SOAKS": "failure"}).returncode - != 0 - ) + assert outcome_failures(outcomes, "ib") == ["provider-soaks"] all_outcomes = { **outcomes, - "EXTENDED_PROVIDER": "all", - "FEED_EVIDENCE": "success", - "PAPER_EVIDENCE": "success", - "PROVIDER_SOAKS": "success", + "feed-evidence": "success", + "paper-evidence": "success", + "provider-soaks": "success", } - assert run_gate(all_outcomes).returncode == 0 - for variable in ("FEED_EVIDENCE", "PAPER_EVIDENCE", "PROVIDER_SOAKS"): + assert outcome_failures(all_outcomes, "all") == [] + for variable in ("feed-evidence", "paper-evidence", "provider-soaks"): seeded = {**all_outcomes, variable: "failure"} - assert run_gate(seeded).returncode != 0, variable + assert outcome_failures(seeded, "all") == [variable] def test_paper_qualification_uses_a_clean_explicit_runtime() -> None: