Phase 1: fail closed across verifier evidence paths

.github/workflows/agents-shipgate-self.yml

@@ -1,3 +1,3 @@
 name: Agents Shipgate Self Dogfood
 
 on:
@@ -21,6 +21,6 @@
           verify_mode: verify
           ci_mode: advisory
           diff_base: target
-          fail_on_merge_verdicts: blocked
+          fail_on_merge_verdicts: "blocked,human_review_required,insufficient_evidence,unknown"
           upload_artifact: "true"
           pr_comment: "false"

.github/workflows/agents-shipgate.yml

@@ -1,3 +1,3 @@
 name: Agents Shipgate
 
 on:
@@ -20,6 +20,6 @@
           config: shipgate.yaml
           ci_mode: advisory
           diff_base: target
-          fail_on_merge_verdicts: blocked
+          fail_on_merge_verdicts: "blocked,human_review_required,insufficient_evidence,unknown"
           upload_artifact: "true"
           pr_comment: "false"

llms.txt

@@ -102,7 +102,7 @@
 - Before editing protected surfaces, run `agents-shipgate preflight --workspace . --plan - --json` with a `PreflightPlanV1` object; stop when `requires_human_review` is true.
 - Configure the minimal preview route: `agents-shipgate init --workspace . --write --json`. CI and agent-instruction trust roots are separate reviewed changes.
 - Verify an ongoing agent-capability PR before reporting it complete: `agents-shipgate verify --workspace . --config shipgate.yaml --base origin/main --head HEAD --ci-mode advisory --format json`.
-- Validate `verification-receipt.json` first, then read `agent-handoff.json` and switch on `control.state`; read `gate.merge_verdict`, `gate.can_merge_without_human`, `next_action`, `fix_task`, and `capability_review.top_changes`. The unified control model requires runtime contract 14. Then read `report.json.release_decision.decision`; it is the release gate.
+- Validate `verification-receipt.json` first, then read `agent-handoff.json` and switch on `control.state`; read `gate.merge_verdict`, `gate.can_merge_without_human`, `control.next_action`, `fix_task`, and `capability_review.top_changes`. The unified control model requires runtime contract 14. Then read `report.json.release_decision.decision`; it is the release gate.
 - Run the verify-native demo fixture: `agents-shipgate fixture run ai_generated_refund_pr`.
 - Scan a static repo/report fixture when you need the lower-level report path: `agents-shipgate fixture run support_refund_agent`.
 - Export a stable static capability lock: `agents-shipgate capability export -c shipgate.yaml`.

src/agents_shipgate/ci/release_decision.py

@@ -62,9 +62,7 @@ def evidence_below_ie_threshold(evidence: EvidenceCoverageDecision, *, tool_coun
         evidence.semantic_coverage.gap_count > 0
         or evidence.policy_gap_count > 0
         or any(
-            gap.kind == "source_warning"
-            and gap.next_action.path == "--diff-from"
-            and gap.next_action.command == "agents-shipgate scan -c shipgate.yaml --format json"
+            gap.kind == "source_warning" and gap.next_action.kind == "provide_source"
             for gap in evidence.evidence_gaps
         )
         or evidence.low_confidence_tool_count >= _low_confidence_tool_threshold(tool_count)

src/agents_shipgate/cli/agent_interface.py

@@ -45,7 +45,7 @@ def handoff(
         help="Print the handoff JSON to stdout.",
     ),
 ) -> None:
-    """Render the shipgate.agent_handoff/v3 artifact from verifier outputs."""
+    """Render the shipgate.agent_handoff/v5 artifact from verifier outputs."""
 
     try:
         verifier_payload = _load_required_json(source, "verifier.json")

src/agents_shipgate/cli/scan/decision.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 from typing import Any
 
+from agents_shipgate.checks.plugin_validation import strict_failure_messages
 from agents_shipgate.checks.registry import check_catalog, run_checks
 from agents_shipgate.core.capability_policy import build_capability_policy_subjects
 from agents_shipgate.core.capability_traces import build_capability_runtime_evidence
@@ -121,6 +122,10 @@ def _run_checks_and_decide(
         loaded_plugins=loaded_plugins,
         extra_known_check_ids={resolved.rule.id for resolved in inputs.policy_packs.rules},
     )
+    action_surface_warnings.extend(
+        f"Enabled check plugin failed validation or execution: {message}"
+        for message in strict_failure_messages(loaded_plugins)
+    )
     findings.extend(run_policy_pack_rules(context, inputs.policy_packs))
     findings.extend(
         evaluate_action_surface_policies(

src/agents_shipgate/cli/verify/orchestrator.py

@@ -282,6 +282,7 @@ def run_verify(
             changed_files = _dedupe_paths([*changed_files, *worktree_paths])
             diff_text = _join_diff_text(diff_text, worktree_diff)
         except Exception as exc:  # noqa: BLE001 - local context degrades only.
+            diff_unavailable = True
             base_notes.append(f"Could not collect working-tree diff context: {exc}")
 
     trigger = evaluate(

src/agents_shipgate/inputs/mcp_manifest.py

@@ -391,20 +391,29 @@ def _loaded_source_from_servers(
 
 
 def _load_mcp_json(path: Path, base_dir: Path) -> list[LoadedToolSource]:
+    source_ref = _relative(path, base_dir)
     try:
         data, positions = load_structured_file_with_positions(path)
     except InputParseError as exc:
         return [
             LoadedToolSource(
-                source_id=f"mcp_json:{_relative(path, base_dir)}",
+                source_id=f"mcp_json:{source_ref}",
                 source_type="codex_config_mcp",
                 warnings=[str(exc)],
             )
         ]
     raw_servers = data.get("mcpServers") if isinstance(data, dict) else None
     if not isinstance(raw_servers, dict):
-        return []
-    source_ref = _relative(path, base_dir)
+        return [
+            LoadedToolSource(
+                source_id=f"mcp_json:{source_ref}",
+                source_type="codex_config_mcp",
+                warnings=[
+                    f"Invalid MCP config {source_ref}: expected top-level "
+                    "`mcpServers` to be an object."
+                ],
+            )
+        ]
     source_path = manifest_relative_path(str(path.resolve()), base_dir)
     servers = normalize_mcp_json_servers(
         data,

src/agents_shipgate/inputs/n8n/_tools.py

@@ -272,9 +272,21 @@ def _mcp_client_tools(
             reason="MCP Client Tool exposes All or All Except without a local inventory.",
             warnings=warnings,
         )
+    elif mode == "unknown" and not artifacts.tool_inventory_files:
+        _dynamic(
+            artifacts,
+            kind="mcp_client_selection_mode_unknown",
+            item=item,
+            source_path=source_path,
+            reason=(
+                "MCP Client Tool selection mode is unrecognized; "
+                "static tool exposure cannot be proven."
+            ),
+            warnings=warnings,
+        )
     names = selected or [
         f"{_redact_text(item.name) or item.name}.*"
-        if mode in {"all", "all_except"}
+        if mode in {"all", "all_except", "unknown"}
         else _tool_name(item)
     ]
     tools = [
@@ -290,7 +302,7 @@ def _mcp_client_tools(
         )
         for name in names
     ]
-    if mode in {"all", "all_except"}:
+    if mode in {"all", "all_except", "unknown"}:
         for tool in tools:
             tool.annotations["wildcard_tools"] = True
             tool.annotations["tool_selection_mode"] = mode
@@ -445,7 +457,9 @@ def _selection_mode(parameters: dict[str, Any]) -> str:
     if normalized in {"selected", "selected_tools", "specific"}:
         return "selected"
     selected = _selected_mcp_tools(parameters)
-    return "selected" if selected else "unknown"
+    if selected:
+        return "selected"
+    return "unknown" if value else "unspecified"
 
 
 def _is_unfiltered_mode(parameters: dict[str, Any]) -> bool:

tests/test_action_metadata.py

@@ -222,7 +222,10 @@ def test_agents_shipgate_workflow_uses_merge_verdict_policy_input():
     text = (WORKFLOW_DIR / "agents-shipgate.yml").read_text(encoding="utf-8")
 
     assert "fail_on_decisions" not in text
-    assert "fail_on_merge_verdicts: blocked" in text
+    assert (
+        'fail_on_merge_verdicts: "blocked,human_review_required,'
+        'insufficient_evidence,unknown"' in text
+    )
 
 
 def test_action_step_summary_leads_with_verifier_merge_state():

tests/test_mcp_manifest.py

@@ -113,6 +113,29 @@ def test_mcp_json_stub_becomes_wildcard_unknown_tool(tmp_path: Path) -> None:
     assert tool.annotations["mcp_unknown_schema"] is True
 
 
+@pytest.mark.parametrize(
+    "payload",
+    [
+        {},
+        {"mcpServers": []},
+        {"mcpServers": "payments"},
+    ],
+)
+def test_mcp_json_wrong_shape_emits_source_warning(
+    tmp_path: Path, payload: dict[str, object]
+) -> None:
+    (tmp_path / ".mcp.json").write_text(json.dumps(payload), encoding="utf-8")
+
+    loaded = load_codex_config_mcp_sources(tmp_path, tmp_path)
+
+    assert len(loaded) == 1
+    assert loaded[0].tools == []
+    assert loaded[0].source_id == "mcp_json:.mcp.json"
+    assert loaded[0].warnings == [
+        "Invalid MCP config .mcp.json: expected top-level `mcpServers` to be an object."
+    ]
+
+
 def test_mcp_json_sources_strip_reserved_binding_annotations(tmp_path: Path) -> None:
     (tmp_path / ".mcp.json").write_text(
         json.dumps(

tests/test_n8n.py

@@ -193,6 +193,48 @@ def test_n8n_static_extraction_and_findings(tmp_path):
     assert {finding.agent_action for finding in n8n_findings} == {"escalate_to_human"}
 
 
+def test_n8n_unknown_mcp_selection_mode_degrades_evidence(tmp_path):
+    project = tmp_path / "project"
+    project.mkdir()
+    workflow = _workflow()
+    mcp_client = next(node for node in workflow["nodes"] if node["id"] == "mcp-client")
+    mcp_client["parameters"] = {"toolsToInclude": "Future Typo Mode"}
+    (project / "workflow.json").write_text(json.dumps(workflow), encoding="utf-8")
+    (project / "shipgate.yaml").write_text(
+        """
+version: "0.1"
+project:
+  name: n8n-unknown-mcp-selection
+agent:
+  name: n8n-agent
+environment:
+  target: production_like
+n8n:
+  workflows:
+    - path: workflow.json
+""",
+        encoding="utf-8",
+    )
+
+    report, _ = run_scan(
+        config_path=project / "shipgate.yaml",
+        output_dir=tmp_path / "reports",
+        formats=["json"],
+        ci_mode="advisory",
+    )
+
+    assert any("selection mode is unrecognized" in warning for warning in report.source_warnings)
+    dynamic = [
+        finding
+        for finding in report.findings
+        if finding.check_id == "SHIP-N8N-DYNAMIC-TOOL-SURFACE-NOT-ENUMERABLE"
+        and finding.evidence["surface"]["kind"] == "mcp_client_selection_mode_unknown"
+    ]
+    assert len(dynamic) == 1
+    assert any(tool["name"] == "MCP Client.*" for tool in report.tool_inventory)
+    assert report.release_decision.decision != "passed"
+
+
 def test_n8n_mcp_server_trigger_exposed_tools_are_normalized_as_mcp(tmp_path):
     project = tmp_path / "project"
     project.mkdir()

tests/test_plugin_validation.py

@@ -82,6 +82,12 @@ def boom():
     assert any("simulated module load failure" in err for err in record["validation_errors"])
     assert record["check_id"] is None
     assert record["runtime_errors"] == []
+    assert any(
+        "Enabled check plugin failed validation or execution" in warning
+        and "simulated module load failure" in warning
+        for warning in report.source_warnings
+    )
+    assert report.release_decision.decision != "passed"
 
 
 # --- Gate 2: signature ------------------------------------------------------
@@ -135,6 +141,10 @@ def with_extras(context, *, config=None):
         ci_mode="advisory",
     )
     assert report.loaded_plugins[0]["validation_status"] == VALID
+    assert not any(
+        warning.startswith("Enabled check plugin failed validation or execution")
+        for warning in report.source_warnings
+    )
 
 
 def test_gate_signature_rejects_required_keyword_only(monkeypatch, tmp_path):
@@ -570,6 +580,12 @@ def plugin(context):
     assert any("simulated runtime crash" in err for err in record["runtime_errors"])
     # Other findings (built-ins) still ran.
     assert exit_code == 0
+    assert any(
+        "Enabled check plugin failed validation or execution" in warning
+        and "simulated runtime crash" in warning
+        for warning in report.source_warnings
+    )
+    assert report.release_decision.decision != "passed"
 
 
 def test_runtime_plugin_must_return_list(monkeypatch, tmp_path):

tests/test_public_surface_contract.py

@@ -2065,7 +2065,10 @@ def test_self_dogfood_manifest_scans_codex_plugin_package() -> None:
     workflow = _read(".github/workflows/agents-shipgate-self.yml")
     assert "config: shipgate-self.yaml" in workflow
     assert "verify_mode: verify" in workflow
-    assert "fail_on_merge_verdicts: blocked" in workflow
+    assert (
+        'fail_on_merge_verdicts: "blocked,human_review_required,'
+        'insufficient_evidence,unknown"' in workflow
+    )
 
 
 def test_pre_commit_local_docs_show_same_path_trigger_clauses():

tests/test_release_decision.py

@@ -10,6 +10,7 @@
     _LOW_CONFIDENCE_TOOL_RATIO,
     _MAX_TOLERATED_SOURCE_WARNINGS,
     build_release_decision,
+    evidence_below_ie_threshold,
 )
 from agents_shipgate.core.domain import (
     AuthInfo,
@@ -376,6 +377,29 @@ def test_one_source_warning_is_review_required():
     assert "source-loader" in decision.reason
 
 
+def test_diff_reference_degradation_uses_typed_action_kind_for_ie():
+    warning = (
+        "Base report predates report schema semantic evidence and is not "
+        "comparable with --diff-from. Regenerate it."
+    )
+    tool = _tool(confidence="high")
+    decision = _build(
+        _report(tools=[tool], source_warnings=[warning]),
+        ci_mode="strict",
+        tools=[tool],
+    )
+
+    assert decision.decision == "insufficient_evidence"
+    [gap] = decision.evidence_coverage.evidence_gaps
+    assert gap.kind == "source_warning"
+    assert gap.next_action.kind == "provide_source"
+
+    # Routing remains typed even if presentation strings change.
+    gap.next_action.command = None
+    gap.next_action.path = None
+    assert evidence_below_ie_threshold(decision.evidence_coverage, tool_count=1) is True
+
+
 def test_zero_source_warnings_clean_scan_passes():
     """Regression guard: 0 warnings + 0 findings + all-high-confidence
     tools must still produce passed under the new logic."""

tests/test_verify_orchestrator.py

@@ -232,6 +232,54 @@ def test_verify_threads_uncommitted_worktree_files_into_head_scan(tmp_path):
     )
 
 
+def test_verify_fails_closed_when_worktree_diff_cannot_be_collected(monkeypatch, tmp_path):
+    repo = tmp_path / "repo"
+    sample_dst = repo / "samples" / "support_refund_agent"
+    sample_dst.parent.mkdir(parents=True)
+    shutil.copytree(REPO_ROOT / "samples" / "support_refund_agent", sample_dst)
+
+    _git(repo, "init")
+    _git(repo, "config", "user.email", "test@example.test")
+    _git(repo, "config", "user.name", "Test User")
+    _git(repo, "add", ".")
+    _git(repo, "commit", "-m", "base")
+
+    def fail_worktree_context(_repo):
+        raise RuntimeError("simulated worktree diff failure")
+
+    monkeypatch.setattr(
+        "agents_shipgate.cli.verify.orchestrator.working_tree_context",
+        fail_worktree_context,
+    )
+
+    verifier, report, exit_code = run_verify(
+        workspace=repo,
+        config=Path("samples/support_refund_agent/shipgate.yaml"),
+        base="HEAD",
+        head="HEAD",
+        archive_head=False,
+        out=repo / "agents-shipgate-reports",
+        ci_mode="advisory",
+        fail_on=None,
+        baseline=None,
+        baseline_mode="new-findings",
+        diff_from=None,
+        policy_packs=None,
+        plugins_enabled=False,
+        strict_plugins=False,
+        suggest_patches=False,
+        no_heuristics=False,
+        verbose=False,
+    )
+
+    assert exit_code == 2
+    assert report is None
+    assert verifier.head_status == "failed"
+    assert verifier.merge_verdict == "unknown"
+    assert verifier.can_merge_without_human is False
+    assert any("simulated worktree diff failure" in note for note in verifier.base_notes)
+
+
 def test_verify_pr232_toolkit_bound_removal_blocks(tmp_path):
     """Stripe stripe/ai PR #232: the agent's Stripe tools load via a dynamic
     factory the static extractor cannot enumerate, so the head scan alone reads