diff --git a/.github/workflows/agents-shipgate-self.yml b/.github/workflows/agents-shipgate-self.yml index 5b9b9ac0..ad81e7be 100644 --- a/.github/workflows/agents-shipgate-self.yml +++ b/.github/workflows/agents-shipgate-self.yml @@ -21,6 +21,6 @@ jobs: verify_mode: verify ci_mode: advisory diff_base: target - fail_on_merge_verdicts: blocked + fail_on_merge_verdicts: "blocked,human_review_required,insufficient_evidence,unknown" upload_artifact: "true" pr_comment: "false" diff --git a/.github/workflows/agents-shipgate.yml b/.github/workflows/agents-shipgate.yml index 32de020b..225244c2 100644 --- a/.github/workflows/agents-shipgate.yml +++ b/.github/workflows/agents-shipgate.yml @@ -20,6 +20,6 @@ jobs: config: shipgate.yaml ci_mode: advisory diff_base: target - fail_on_merge_verdicts: blocked + fail_on_merge_verdicts: "blocked,human_review_required,insufficient_evidence,unknown" upload_artifact: "true" pr_comment: "false" diff --git a/llms.txt b/llms.txt index c97b9c16..14aea41d 100644 --- a/llms.txt +++ b/llms.txt @@ -102,7 +102,7 @@ - Before editing protected surfaces, run `agents-shipgate preflight --workspace . --plan - --json` with a `PreflightPlanV1` object; stop when `requires_human_review` is true. - Configure the minimal preview route: `agents-shipgate init --workspace . --write --json`. CI and agent-instruction trust roots are separate reviewed changes. - Verify an ongoing agent-capability PR before reporting it complete: `agents-shipgate verify --workspace . --config shipgate.yaml --base origin/main --head HEAD --ci-mode advisory --format json`. -- Validate `verification-receipt.json` first, then read `agent-handoff.json` and switch on `control.state`; read `gate.merge_verdict`, `gate.can_merge_without_human`, `next_action`, `fix_task`, and `capability_review.top_changes`. The unified control model requires runtime contract 14. Then read `report.json.release_decision.decision`; it is the release gate. +- Validate `verification-receipt.json` first, then read `agent-handoff.json` and switch on `control.state`; read `gate.merge_verdict`, `gate.can_merge_without_human`, `control.next_action`, `fix_task`, and `capability_review.top_changes`. The unified control model requires runtime contract 14. Then read `report.json.release_decision.decision`; it is the release gate. - Run the verify-native demo fixture: `agents-shipgate fixture run ai_generated_refund_pr`. - Scan a static repo/report fixture when you need the lower-level report path: `agents-shipgate fixture run support_refund_agent`. - Export a stable static capability lock: `agents-shipgate capability export -c shipgate.yaml`. diff --git a/src/agents_shipgate/ci/release_decision.py b/src/agents_shipgate/ci/release_decision.py index 1b1ad07f..2784b11e 100644 --- a/src/agents_shipgate/ci/release_decision.py +++ b/src/agents_shipgate/ci/release_decision.py @@ -62,9 +62,7 @@ def evidence_below_ie_threshold(evidence: EvidenceCoverageDecision, *, tool_coun evidence.semantic_coverage.gap_count > 0 or evidence.policy_gap_count > 0 or any( - gap.kind == "source_warning" - and gap.next_action.path == "--diff-from" - and gap.next_action.command == "agents-shipgate scan -c shipgate.yaml --format json" + gap.kind == "source_warning" and gap.next_action.kind == "provide_source" for gap in evidence.evidence_gaps ) or evidence.low_confidence_tool_count >= _low_confidence_tool_threshold(tool_count) diff --git a/src/agents_shipgate/cli/agent_interface.py b/src/agents_shipgate/cli/agent_interface.py index 325df21c..a16fbebc 100644 --- a/src/agents_shipgate/cli/agent_interface.py +++ b/src/agents_shipgate/cli/agent_interface.py @@ -45,7 +45,7 @@ def handoff( help="Print the handoff JSON to stdout.", ), ) -> None: - """Render the shipgate.agent_handoff/v3 artifact from verifier outputs.""" + """Render the shipgate.agent_handoff/v5 artifact from verifier outputs.""" try: verifier_payload = _load_required_json(source, "verifier.json") diff --git a/src/agents_shipgate/cli/scan/decision.py b/src/agents_shipgate/cli/scan/decision.py index 3573ac8c..83d4ce55 100644 --- a/src/agents_shipgate/cli/scan/decision.py +++ b/src/agents_shipgate/cli/scan/decision.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any +from agents_shipgate.checks.plugin_validation import strict_failure_messages from agents_shipgate.checks.registry import check_catalog, run_checks from agents_shipgate.core.capability_policy import build_capability_policy_subjects from agents_shipgate.core.capability_traces import build_capability_runtime_evidence @@ -121,6 +122,10 @@ def _run_checks_and_decide( loaded_plugins=loaded_plugins, extra_known_check_ids={resolved.rule.id for resolved in inputs.policy_packs.rules}, ) + action_surface_warnings.extend( + f"Enabled check plugin failed validation or execution: {message}" + for message in strict_failure_messages(loaded_plugins) + ) findings.extend(run_policy_pack_rules(context, inputs.policy_packs)) findings.extend( evaluate_action_surface_policies( diff --git a/src/agents_shipgate/cli/verify/orchestrator.py b/src/agents_shipgate/cli/verify/orchestrator.py index 04373824..3c704797 100644 --- a/src/agents_shipgate/cli/verify/orchestrator.py +++ b/src/agents_shipgate/cli/verify/orchestrator.py @@ -282,6 +282,7 @@ def run_verify( changed_files = _dedupe_paths([*changed_files, *worktree_paths]) diff_text = _join_diff_text(diff_text, worktree_diff) except Exception as exc: # noqa: BLE001 - local context degrades only. + diff_unavailable = True base_notes.append(f"Could not collect working-tree diff context: {exc}") trigger = evaluate( diff --git a/src/agents_shipgate/inputs/mcp_manifest.py b/src/agents_shipgate/inputs/mcp_manifest.py index 5256590c..37394b30 100644 --- a/src/agents_shipgate/inputs/mcp_manifest.py +++ b/src/agents_shipgate/inputs/mcp_manifest.py @@ -391,20 +391,29 @@ def _loaded_source_from_servers( def _load_mcp_json(path: Path, base_dir: Path) -> list[LoadedToolSource]: + source_ref = _relative(path, base_dir) try: data, positions = load_structured_file_with_positions(path) except InputParseError as exc: return [ LoadedToolSource( - source_id=f"mcp_json:{_relative(path, base_dir)}", + source_id=f"mcp_json:{source_ref}", source_type="codex_config_mcp", warnings=[str(exc)], ) ] raw_servers = data.get("mcpServers") if isinstance(data, dict) else None if not isinstance(raw_servers, dict): - return [] - source_ref = _relative(path, base_dir) + return [ + LoadedToolSource( + source_id=f"mcp_json:{source_ref}", + source_type="codex_config_mcp", + warnings=[ + f"Invalid MCP config {source_ref}: expected top-level " + "`mcpServers` to be an object." + ], + ) + ] source_path = manifest_relative_path(str(path.resolve()), base_dir) servers = normalize_mcp_json_servers( data, diff --git a/src/agents_shipgate/inputs/n8n/_tools.py b/src/agents_shipgate/inputs/n8n/_tools.py index 82b20e41..37bcbe9f 100644 --- a/src/agents_shipgate/inputs/n8n/_tools.py +++ b/src/agents_shipgate/inputs/n8n/_tools.py @@ -272,9 +272,21 @@ def _mcp_client_tools( reason="MCP Client Tool exposes All or All Except without a local inventory.", warnings=warnings, ) + elif mode == "unknown" and not artifacts.tool_inventory_files: + _dynamic( + artifacts, + kind="mcp_client_selection_mode_unknown", + item=item, + source_path=source_path, + reason=( + "MCP Client Tool selection mode is unrecognized; " + "static tool exposure cannot be proven." + ), + warnings=warnings, + ) names = selected or [ f"{_redact_text(item.name) or item.name}.*" - if mode in {"all", "all_except"} + if mode in {"all", "all_except", "unknown"} else _tool_name(item) ] tools = [ @@ -290,7 +302,7 @@ def _mcp_client_tools( ) for name in names ] - if mode in {"all", "all_except"}: + if mode in {"all", "all_except", "unknown"}: for tool in tools: tool.annotations["wildcard_tools"] = True tool.annotations["tool_selection_mode"] = mode @@ -445,7 +457,9 @@ def _selection_mode(parameters: dict[str, Any]) -> str: if normalized in {"selected", "selected_tools", "specific"}: return "selected" selected = _selected_mcp_tools(parameters) - return "selected" if selected else "unknown" + if selected: + return "selected" + return "unknown" if value else "unspecified" def _is_unfiltered_mode(parameters: dict[str, Any]) -> bool: diff --git a/tests/test_action_metadata.py b/tests/test_action_metadata.py index ee1f8d47..37b9567f 100644 --- a/tests/test_action_metadata.py +++ b/tests/test_action_metadata.py @@ -222,7 +222,10 @@ def test_agents_shipgate_workflow_uses_merge_verdict_policy_input(): text = (WORKFLOW_DIR / "agents-shipgate.yml").read_text(encoding="utf-8") assert "fail_on_decisions" not in text - assert "fail_on_merge_verdicts: blocked" in text + assert ( + 'fail_on_merge_verdicts: "blocked,human_review_required,' + 'insufficient_evidence,unknown"' in text + ) def test_action_step_summary_leads_with_verifier_merge_state(): diff --git a/tests/test_mcp_manifest.py b/tests/test_mcp_manifest.py index fb0ae924..cbe91b13 100644 --- a/tests/test_mcp_manifest.py +++ b/tests/test_mcp_manifest.py @@ -113,6 +113,29 @@ def test_mcp_json_stub_becomes_wildcard_unknown_tool(tmp_path: Path) -> None: assert tool.annotations["mcp_unknown_schema"] is True +@pytest.mark.parametrize( + "payload", + [ + {}, + {"mcpServers": []}, + {"mcpServers": "payments"}, + ], +) +def test_mcp_json_wrong_shape_emits_source_warning( + tmp_path: Path, payload: dict[str, object] +) -> None: + (tmp_path / ".mcp.json").write_text(json.dumps(payload), encoding="utf-8") + + loaded = load_codex_config_mcp_sources(tmp_path, tmp_path) + + assert len(loaded) == 1 + assert loaded[0].tools == [] + assert loaded[0].source_id == "mcp_json:.mcp.json" + assert loaded[0].warnings == [ + "Invalid MCP config .mcp.json: expected top-level `mcpServers` to be an object." + ] + + def test_mcp_json_sources_strip_reserved_binding_annotations(tmp_path: Path) -> None: (tmp_path / ".mcp.json").write_text( json.dumps( diff --git a/tests/test_n8n.py b/tests/test_n8n.py index 88a8db41..d187ee56 100644 --- a/tests/test_n8n.py +++ b/tests/test_n8n.py @@ -193,6 +193,48 @@ def test_n8n_static_extraction_and_findings(tmp_path): assert {finding.agent_action for finding in n8n_findings} == {"escalate_to_human"} +def test_n8n_unknown_mcp_selection_mode_degrades_evidence(tmp_path): + project = tmp_path / "project" + project.mkdir() + workflow = _workflow() + mcp_client = next(node for node in workflow["nodes"] if node["id"] == "mcp-client") + mcp_client["parameters"] = {"toolsToInclude": "Future Typo Mode"} + (project / "workflow.json").write_text(json.dumps(workflow), encoding="utf-8") + (project / "shipgate.yaml").write_text( + """ +version: "0.1" +project: + name: n8n-unknown-mcp-selection +agent: + name: n8n-agent +environment: + target: production_like +n8n: + workflows: + - path: workflow.json +""", + encoding="utf-8", + ) + + report, _ = run_scan( + config_path=project / "shipgate.yaml", + output_dir=tmp_path / "reports", + formats=["json"], + ci_mode="advisory", + ) + + assert any("selection mode is unrecognized" in warning for warning in report.source_warnings) + dynamic = [ + finding + for finding in report.findings + if finding.check_id == "SHIP-N8N-DYNAMIC-TOOL-SURFACE-NOT-ENUMERABLE" + and finding.evidence["surface"]["kind"] == "mcp_client_selection_mode_unknown" + ] + assert len(dynamic) == 1 + assert any(tool["name"] == "MCP Client.*" for tool in report.tool_inventory) + assert report.release_decision.decision != "passed" + + def test_n8n_mcp_server_trigger_exposed_tools_are_normalized_as_mcp(tmp_path): project = tmp_path / "project" project.mkdir() diff --git a/tests/test_plugin_validation.py b/tests/test_plugin_validation.py index 2f1a6141..c41f8bb1 100644 --- a/tests/test_plugin_validation.py +++ b/tests/test_plugin_validation.py @@ -82,6 +82,12 @@ def boom(): assert any("simulated module load failure" in err for err in record["validation_errors"]) assert record["check_id"] is None assert record["runtime_errors"] == [] + assert any( + "Enabled check plugin failed validation or execution" in warning + and "simulated module load failure" in warning + for warning in report.source_warnings + ) + assert report.release_decision.decision != "passed" # --- Gate 2: signature ------------------------------------------------------ @@ -135,6 +141,10 @@ def with_extras(context, *, config=None): ci_mode="advisory", ) assert report.loaded_plugins[0]["validation_status"] == VALID + assert not any( + warning.startswith("Enabled check plugin failed validation or execution") + for warning in report.source_warnings + ) def test_gate_signature_rejects_required_keyword_only(monkeypatch, tmp_path): @@ -570,6 +580,12 @@ def plugin(context): assert any("simulated runtime crash" in err for err in record["runtime_errors"]) # Other findings (built-ins) still ran. assert exit_code == 0 + assert any( + "Enabled check plugin failed validation or execution" in warning + and "simulated runtime crash" in warning + for warning in report.source_warnings + ) + assert report.release_decision.decision != "passed" def test_runtime_plugin_must_return_list(monkeypatch, tmp_path): diff --git a/tests/test_public_surface_contract.py b/tests/test_public_surface_contract.py index a4626cad..1bf3810f 100644 --- a/tests/test_public_surface_contract.py +++ b/tests/test_public_surface_contract.py @@ -2065,7 +2065,10 @@ def test_self_dogfood_manifest_scans_codex_plugin_package() -> None: workflow = _read(".github/workflows/agents-shipgate-self.yml") assert "config: shipgate-self.yaml" in workflow assert "verify_mode: verify" in workflow - assert "fail_on_merge_verdicts: blocked" in workflow + assert ( + 'fail_on_merge_verdicts: "blocked,human_review_required,' + 'insufficient_evidence,unknown"' in workflow + ) def test_pre_commit_local_docs_show_same_path_trigger_clauses(): diff --git a/tests/test_release_decision.py b/tests/test_release_decision.py index d2be253e..fec4a0ce 100644 --- a/tests/test_release_decision.py +++ b/tests/test_release_decision.py @@ -10,6 +10,7 @@ _LOW_CONFIDENCE_TOOL_RATIO, _MAX_TOLERATED_SOURCE_WARNINGS, build_release_decision, + evidence_below_ie_threshold, ) from agents_shipgate.core.domain import ( AuthInfo, @@ -376,6 +377,29 @@ def test_one_source_warning_is_review_required(): assert "source-loader" in decision.reason +def test_diff_reference_degradation_uses_typed_action_kind_for_ie(): + warning = ( + "Base report predates report schema semantic evidence and is not " + "comparable with --diff-from. Regenerate it." + ) + tool = _tool(confidence="high") + decision = _build( + _report(tools=[tool], source_warnings=[warning]), + ci_mode="strict", + tools=[tool], + ) + + assert decision.decision == "insufficient_evidence" + [gap] = decision.evidence_coverage.evidence_gaps + assert gap.kind == "source_warning" + assert gap.next_action.kind == "provide_source" + + # Routing remains typed even if presentation strings change. + gap.next_action.command = None + gap.next_action.path = None + assert evidence_below_ie_threshold(decision.evidence_coverage, tool_count=1) is True + + def test_zero_source_warnings_clean_scan_passes(): """Regression guard: 0 warnings + 0 findings + all-high-confidence tools must still produce passed under the new logic.""" diff --git a/tests/test_verify_orchestrator.py b/tests/test_verify_orchestrator.py index c5205b77..fc56079f 100644 --- a/tests/test_verify_orchestrator.py +++ b/tests/test_verify_orchestrator.py @@ -232,6 +232,54 @@ def test_verify_threads_uncommitted_worktree_files_into_head_scan(tmp_path): ) +def test_verify_fails_closed_when_worktree_diff_cannot_be_collected(monkeypatch, tmp_path): + repo = tmp_path / "repo" + sample_dst = repo / "samples" / "support_refund_agent" + sample_dst.parent.mkdir(parents=True) + shutil.copytree(REPO_ROOT / "samples" / "support_refund_agent", sample_dst) + + _git(repo, "init") + _git(repo, "config", "user.email", "test@example.test") + _git(repo, "config", "user.name", "Test User") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "base") + + def fail_worktree_context(_repo): + raise RuntimeError("simulated worktree diff failure") + + monkeypatch.setattr( + "agents_shipgate.cli.verify.orchestrator.working_tree_context", + fail_worktree_context, + ) + + verifier, report, exit_code = run_verify( + workspace=repo, + config=Path("samples/support_refund_agent/shipgate.yaml"), + base="HEAD", + head="HEAD", + archive_head=False, + out=repo / "agents-shipgate-reports", + ci_mode="advisory", + fail_on=None, + baseline=None, + baseline_mode="new-findings", + diff_from=None, + policy_packs=None, + plugins_enabled=False, + strict_plugins=False, + suggest_patches=False, + no_heuristics=False, + verbose=False, + ) + + assert exit_code == 2 + assert report is None + assert verifier.head_status == "failed" + assert verifier.merge_verdict == "unknown" + assert verifier.can_merge_without_human is False + assert any("simulated worktree diff failure" in note for note in verifier.base_notes) + + def test_verify_pr232_toolkit_bound_removal_blocks(tmp_path): """Stripe stripe/ai PR #232: the agent's Stripe tools load via a dynamic factory the static extractor cannot enumerate, so the head scan alone reads