From 6e9b673c102bb220fc849d27d18052df13113b9f Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Fri, 10 Jul 2026 23:05:53 -0700 Subject: [PATCH 1/3] Prove agent-to-tool bindings --- .agents/skills/agents-shipgate/SKILL.md | 2 +- .claude-plugin/marketplace.json | 2 +- .github/workflows/ci.yml | 5 + .well-known/agents-shipgate.json | 23 +- AGENTS.md | 28 +- CHANGELOG.md | 11 + README.md | 16 +- STABILITY.md | 25 +- .../.agents-shipgate-kit-metadata.json | 6 +- adoption-kits/claude-code-skill/SKILL.md | 6 +- .../.agents-shipgate-kit-metadata.json | 3 +- adoption-kits/codex-skill/SKILL.md | 2 +- benchmark/agent-pr-governance/cases.yaml | 16 +- .../results/baseline.v0.2.json | 64 +- benchmark/matrix-codex.yaml | 2 +- benchmark/matrix.yaml | 4 +- benchmark/safety-qualification/README.md | 15 +- docs/INDEX.md | 14 +- docs/agent-autofix-boundary.md | 3 +- docs/agent-contract-current.md | 31 +- docs/agent-handoff-schema.v2.json | 2 +- docs/agent-result-schema.v1.json | 2 +- docs/ai-search-summary.md | 8 +- docs/architecture.md | 12 +- docs/autofix-policy.md | 2 +- docs/baseline.md | 2 +- docs/capability-lock-diff-schema.v0.6.json | 1288 +++ docs/capability-lock-schema.v0.5.json | 1223 +++ docs/capability-standard.md | 6 +- docs/codex-boundary-result-schema.v1.json | 2 +- docs/distribution.md | 2 +- docs/examples.md | 5 +- .../capability-lock-diff.v0.6.example.json | 155 + .../capability-lock.v0.5.example.json | 153 + docs/faq.md | 8 +- ...ernance-benchmark-catalog-schema.v0.2.json | 1 + docs/manifest-v0.1.json | 158 + docs/manifest-v0.1.md | 26 + docs/overview.md | 2 +- docs/packet-schema.v0.10.json | 1988 +++++ docs/passed-verdict-contract.md | 26 +- docs/report-reading-for-agents.md | 6 +- docs/report-schema.v0.31.json | 7096 +++++++++++++++++ docs/report-sensitive-fields.json | 3 + docs/report-v1-consolidation-rc.md | 2 +- .../agent-protocol/expected/block-stop.json | 2 +- .../expected/policy-bypass.json | 2 +- .../agent-protocol/expected/repair-after.json | 2 +- .../expected/repair-before.json | 2 +- llms-full.txt | 61 +- llms.txt | 10 +- .../agents-shipgate/.codex-plugin/plugin.json | 2 +- .../skills/agents-shipgate/SKILL.md | 2 +- .../claude-code/.claude-plugin/plugin.json | 2 +- .../skills/agents-shipgate/SKILL.md | 6 +- pyproject.toml | 2 +- samples/agent_weakens_gate/shipgate.yaml | 9 + samples/ai_generated_refund_pr/shipgate.yaml | 10 + samples/clean_read_only_agent/shipgate.yaml | 9 + samples/hitl_evidence_agent/shipgate.yaml | 9 + .../hitl_evidence_covered_agent/shipgate.yaml | 9 + samples/openai_agents_sdk_agent/README.md | 3 +- .../openai_agents_sdk_agent/agents/root.py | 8 + samples/openai_agents_sdk_agent/shipgate.yaml | 1 + samples/simple_anthropic_agent/shipgate.yaml | 10 + .../simple_crewai_agent/expected/report.json | 913 ++- .../simple_crewai_agent/expected/report.md | 11 +- .../expected/report.json | 587 +- .../simple_langchain_agent/expected/report.md | 11 +- .../expected/report.json | 626 +- .../expected/report.md | 11 +- samples/simple_openai_api_agent/shipgate.yaml | 10 + .../support_refund_agent/expected/packet.html | 2 +- .../support_refund_agent/expected/packet.json | 349 +- .../support_refund_agent/expected/packet.md | 30 +- .../support_refund_agent/expected/report.json | 3570 +++++++-- .../support_refund_agent/expected/report.md | 44 +- samples/support_refund_agent/shipgate.yaml | 18 + scripts/generate_schemas.py | 4 +- scripts/run_governance_benchmark.py | 47 +- scripts/run_safety_qualification.py | 20 + .../verify_safety_qualification_release.py | 9 +- skills/agents-shipgate/SKILL.md | 6 +- src/agents_shipgate/__init__.py | 2 +- src/agents_shipgate/ci/release_decision.py | 146 +- src/agents_shipgate/cli/scan/decision.py | 2 + src/agents_shipgate/cli/scan/final_report.py | 5 + src/agents_shipgate/cli/scan/models.py | 6 + src/agents_shipgate/cli/scan/run_identity.py | 7 + src/agents_shipgate/cli/scan/sanitization.py | 45 + src/agents_shipgate/cli/scan/tools_agent.py | 32 +- src/agents_shipgate/core/agent_bindings.py | 858 ++ src/agents_shipgate/core/capabilities.py | 10 + src/agents_shipgate/core/capability_delta.py | 2 + src/agents_shipgate/core/capability_lock.py | 17 +- src/agents_shipgate/core/context.py | 3 + src/agents_shipgate/core/domain.py | 39 +- .../core/findings/report_builder.py | 14 + .../core/findings/summaries.py | 5 + .../core/lenses/tool_surface.py | 20 +- .../core/semantic_assessment.py | 29 + .../core/semantic_consistency.py | 21 + src/agents_shipgate/inputs/crewai.py | 21 +- src/agents_shipgate/inputs/google_adk.py | 20 +- src/agents_shipgate/inputs/langchain.py | 28 +- src/agents_shipgate/inputs/n8n/_tools.py | 29 +- src/agents_shipgate/inputs/n8n/_workflows.py | 6 + .../inputs/openai_sdk_static.py | 154 +- src/agents_shipgate/packet/json_packet.py | 26 +- src/agents_shipgate/report/markdown.py | 40 +- src/agents_shipgate/report/summary_text.py | 7 + src/agents_shipgate/schemas/bindings.py | 112 + src/agents_shipgate/schemas/capabilities.py | 11 +- .../schemas/capability_semantics.py | 1 + src/agents_shipgate/schemas/contract.py | 3 +- .../schemas/manifest/__init__.py | 10 + .../schemas/manifest/agent_bindings.py | 116 + src/agents_shipgate/schemas/manifest/root.py | 2 + src/agents_shipgate/schemas/packet.py | 2 +- src/agents_shipgate/schemas/report.py | 40 +- .../schemas/safety_qualification.py | 87 +- src/agents_shipgate/schemas/semantic.py | 34 +- src/agents_shipgate/schemas/surfaces.py | 2 +- .../claude-code-block-stop.json | 2 +- .../agent_protocol/codex-block-stop.json | 2 +- .../agent_protocol/codex-repair-after.json | 2 +- .../agent_protocol/codex-repair-before.json | 2 +- .../agent_protocol/cursor-block-stop.json | 2 +- .../agents_requirement_removed.json | 2 +- .../codex_boundary_result/docs_only.json | 2 +- .../github_action_removed.json | 2 +- .../codex_boundary_result/malformed_toml.json | 2 +- .../mcp_auto_approve_write.json | 2 +- .../network_wildcard.json | 2 +- .../python_refactor.json | 2 +- .../unknown_permission_key.json | 2 +- tests/test_action_surface_diff.py | 28 + tests/test_agent_bindings.py | 165 + tests/test_agent_instructions_apply.py | 2 +- tests/test_agent_instructions_renderers.py | 8 +- tests/test_agent_mode.py | 4 +- tests/test_anthropic_api.py | 7 + ...st_capability_change_schema_hash_parity.py | 9 + tests/test_capability_domain.py | 2 +- tests/test_capability_lock.py | 11 +- tests/test_cli.py | 19 +- tests/test_codex_plugin_launch_package.py | 7 + tests/test_crewai.py | 4 +- tests/test_evidence_backed_pass.py | 12 + tests/test_evidence_packet.py | 16 +- tests/test_fingerprint_compatibility.py | 3 +- tests/test_fixture_no_import.py | 6 +- tests/test_google_adk.py | 3 +- tests/test_governance_benchmark.py | 12 +- tests/test_langchain.py | 9 +- tests/test_large_sample.py | 60 +- tests/test_latency_budget.py | 4 +- tests/test_manifest_consistency.py | 14 + tests/test_mcp_server.py | 2 +- tests/test_miner_constructed.py | 18 +- tests/test_n8n.py | 33 +- tests/test_openai_api.py | 5 +- tests/test_p0_binding_canaries.py | 118 + tests/test_p0_safety_canaries.py | 14 + tests/test_policy_packs.py | 38 +- tests/test_privacy.py | 20 + tests/test_release_decision.py | 10 + tests/test_reports.py | 8 + tests/test_safety_qualification.py | 21 +- tests/test_safety_qualification_release.py | 21 +- tests/test_scan.py | 131 +- tests/test_scenario_suggest.py | 8 +- tests/test_schema_boundaries.py | 7 +- tests/test_semantic_cold_start.py | 11 +- tests/test_tool_identity.py | 8 + tests/test_toolkit_bounds_check.py | 8 +- tests/test_v07_metadata_roundtrip.py | 4 +- tests/test_validation_evidence.py | 8 + tests/test_verifier_scenarios.py | 27 +- tests/test_verify.py | 4 +- tests/test_verify_config_binding.py | 6 +- 181 files changed, 20651 insertions(+), 1209 deletions(-) create mode 100644 docs/capability-lock-diff-schema.v0.6.json create mode 100644 docs/capability-lock-schema.v0.5.json create mode 100644 docs/examples/capability-lock-diff.v0.6.example.json create mode 100644 docs/examples/capability-lock.v0.5.example.json create mode 100644 docs/packet-schema.v0.10.json create mode 100644 docs/report-schema.v0.31.json create mode 100644 samples/openai_agents_sdk_agent/agents/root.py create mode 100644 src/agents_shipgate/core/agent_bindings.py create mode 100644 src/agents_shipgate/schemas/bindings.py create mode 100644 src/agents_shipgate/schemas/manifest/agent_bindings.py create mode 100644 tests/test_agent_bindings.py create mode 100644 tests/test_p0_binding_canaries.py diff --git a/.agents/skills/agents-shipgate/SKILL.md b/.agents/skills/agents-shipgate/SKILL.md index 80107eb8..7c8caf80 100644 --- a/.agents/skills/agents-shipgate/SKILL.md +++ b/.agents/skills/agents-shipgate/SKILL.md @@ -21,7 +21,7 @@ Do not use it for general linting, runtime monitoring, evals, model-output quali 6. For local agent control, run `shipgate check --agent codex --workspace . --format codex-boundary-json` and read the stdout `shipgate.codex_boundary_result/v1` object. Switch on `decision`; follow `first_next_action`, `repair`, and `human_review`. 7. Before editing `shipgate.yaml`, Shipgate CI, AGENTS/CLAUDE/Cursor rules, policy packs, baselines, waivers, suppressions, Codex hooks/config, Codex plugin manifests, `.mcp.json`, `.app.json`, or `SKILL.md`, plan to run `agents-shipgate verify` before completion and route trust-root review to a human when the verifier requires it. 8. For full PR verification, read `agents-shipgate-reports/agent-handoff.json` first, then `verifier.json` for detailed controller state, then `verify-run.json` for reproducibility metadata, then `report.json` for reviewer detail; `report.json.release_decision.decision` remains the release gate. -9. Auto-apply only high-confidence safe patches. Do not auto-assert action effect, action authority, approval, confirmation, idempotency, broad-scope, prohibited-action, or runtime-trace evidence. +9. Auto-apply only high-confidence safe patches. Do not auto-assert action effect, action authority, agent bindings, approval, confirmation, idempotency, broad-scope, prohibited-action, or runtime-trace evidence. 10. Ensure `.gitignore` covers `agents-shipgate-reports/` before committing. ## Fast Paths diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 2f30d952..5957d980 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "name": "agents-shipgate", "source": "./plugins/claude-code", "description": "Run Agents Shipgate Tool-Use Readiness workflows from Claude Code: detect whether Shipgate applies, bootstrap the advisory gate, verify agent-capability PRs, and triage findings without fabricating human approval. Requires the agents-shipgate CLI (pipx install agents-shipgate, contract v10 or newer).", - "version": "0.16.0b1", + "version": "0.16.0b2", "author": { "name": "Three Moons Lab", "url": "https://threemoonslab.com/" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46279394..73531c32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,11 @@ jobs: # verdict regression should fail with a named canary before coverage, # integration, or performance noise can obscure the safety signal. + - name: Agent binding safety canaries (48/48 exact outcomes) + run: python -m pytest tests/test_p0_binding_canaries.py -q + # Binding is a separate trust dimension from effect and authority. + # Keep these exact outcomes visible before the aggregate coverage run. + - name: Test # v0.21 (E7): bumped from 75 → 85. Actual aggregate coverage on # current main is ~88%, so the gate is +10pp tighter with ~3pp diff --git a/.well-known/agents-shipgate.json b/.well-known/agents-shipgate.json index 64875dc6..5c15a7f3 100644 --- a/.well-known/agents-shipgate.json +++ b/.well-known/agents-shipgate.json @@ -3,7 +3,7 @@ "name": "agents-shipgate", "display_name": "Agents Shipgate", "tagline": "The deterministic merge gate for AI-generated agent capability changes", - "version": "0.16.0b1", + "version": "0.16.0b2", "license": "Apache-2.0", "publisher": { "name": "Three Moons Lab", @@ -158,14 +158,14 @@ ], "codex_boundary_result_schema_version": "shipgate.codex_boundary_result/v1", "codex_boundary_result_schema_path": "docs/codex-boundary-result-schema.v1.json", - "report_schema_version": "0.30", - "packet_schema_version": "0.9", + "report_schema_version": "0.31", + "packet_schema_version": "0.10", "verifier_schema_version": "0.2", "verify_run_schema_version": "shipgate.verify_run/v1", "agent_handoff_schema_version": "shipgate.agent_handoff/v2", "agent_handoff_schema_path": "docs/agent-handoff-schema.v2.json", "agent_handoff_artifact": "agents-shipgate-reports/agent-handoff.json", - "contract_version": "12", + "contract_version": "13", "inputs": [ "mcp", "openapi", @@ -225,14 +225,14 @@ "scenario": "scenario.json", "governance_benchmark_result": "benchmark/agent-pr-governance/results/result.json" }, - "capability_lock_schema_version": "0.4", - "capability_lock_diff_schema_version": "0.5", + "capability_lock_schema_version": "0.5", + "capability_lock_diff_schema_version": "0.6", "preflight_schema_version": "0.2", "attestation_schema_version": "0.4", "registry_schema_version": "0.3", "org_evidence_bundle_schema_version": "shipgate.org_evidence_bundle/v1", "host_grants_inventory_schema_version": "0.1", - "capability_standard_version": "0.3", + "capability_standard_version": "0.4", "governance_benchmark_catalog_schema_version": "0.2", "governance_benchmark_result_schema_version": "0.2", "supporting_provisional_surfaces": [ @@ -345,6 +345,7 @@ "do_not_auto_assert": [ "action_effect", "action_authority", + "agent_binding", "approval", "confirmation", "idempotency", @@ -363,16 +364,16 @@ "static_verdict_disclaimer": "This verdict covers deterministic static evidence only. Agents Shipgate did not execute the agent or prove runtime behavior, tool routing, credential enforcement, or safety.", "schemas": { "manifest": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/manifest-v0.1.json", - "report": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/report-schema.v0.30.json", + "report": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/report-schema.v0.31.json", "agent_result": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/agent-result-schema.v1.json", "codex_boundary_result": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/codex-boundary-result-schema.v1.json", "verifier": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/verifier-schema.v0.2.json", "verify_run": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/verify-run-schema.v1.json", "agent_handoff": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/agent-handoff-schema.v2.json", - "packet": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/packet-schema.v0.9.json", + "packet": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/packet-schema.v0.10.json", "preflight": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/preflight-schema.v0.2.json", - "capability_lock": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/capability-lock-schema.v0.4.json", - "capability_lock_diff": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/capability-lock-diff-schema.v0.5.json", + "capability_lock": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/capability-lock-schema.v0.5.json", + "capability_lock_diff": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/capability-lock-diff-schema.v0.6.json", "governance_benchmark_catalog": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/governance-benchmark-catalog-schema.v0.2.json", "governance_benchmark_result": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/governance-benchmark-result-schema.v0.2.json", "feedback": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/feedback-schema.v0.1.json", diff --git a/AGENTS.md b/AGENTS.md index 015bca6a..e0c60deb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -139,9 +139,11 @@ expanding baselines or waivers, removing Shipgate CI, or weakening agent instructions. Verify-mode `SHIP-VERIFY-*` checks make those trust-root edits release-visible and route them to human review. Never invent or auto-fill an action effect or action authority declaration. -Contract v11 publishes these boundaries as `action_effect` and -`action_authority` in `do_not_auto_assert[]`; route semantic next actions to a -human and rerun verification after the reviewed declaration is supplied. +Never invent or auto-fill `agent_bindings` root, tool, or handoff declarations; +they are reviewed closed-world claims about deployed wiring. +Contract v13 publishes these boundaries as `action_effect`, `action_authority`, +and `agent_binding` in `do_not_auto_assert[]`; route binding and semantic next +actions to a human and rerun verification after the reviewed declaration is supplied. To reproduce the verify-native blocked refund PR demo without writing YAML: @@ -343,13 +345,13 @@ restate version archaeology here): - `findings[].{autofix_safe, requires_human_review, suggested_patch_kind, docs_url, provenance_kind, blocks_release}` - `findings[].policy_routing` (policy-pack owner/reviewer/approval routing metadata only; non-enforcing and not part of `evidence`) - `findings[].patches[]` (only when scan ran with `--suggest-patches`) -- `baseline.{matched_count, new_count, resolved_count}` · `tool_inventory[]` · `codex_plugin_surface` +- `baseline.{matched_count, new_count, resolved_count}` · root-reachable `tool_inventory[]` · full `tool_catalog[]` · `codex_plugin_surface` - `action_surface_facts` / `action_surface_diff` -- `release_decision.evidence_coverage.{semantic_coverage,evidence_gaps}` +- `release_decision.evidence_coverage.{binding_coverage,semantic_coverage,evidence_gaps}` - Audit envelopes: `release_decision.contribution_rules[]`, `policy_audit`, `privacy_audit`, `heuristics_filter` — explanatory, never a second gate -The current schema is [`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json). Emitted reports carry `report_schema_version: "0.30"`; v0.29 adds normalized effect/authority assessments and zero-tolerance semantic coverage. A `passed` result now requires complete, conflict-free static evidence for every in-scope action, evaluation of all applicable controls, and no policy condition requiring review. Every release decision explicitly carries `static_analysis_only: true`, `runtime_behavior_verified: false`, and `static_verdict_disclaimer`; packet §1 mirrors them. Semantic gaps are not Findings and cannot be suppressed or baselined. See [`docs/passed-verdict-contract.md`](docs/passed-verdict-contract.md) for the exact claim and [`docs/agent-contract-current.md`](docs/agent-contract-current.md) for version history. v0.29 remains frozen at [`docs/report-schema.v0.29.json`](docs/report-schema.v0.29.json). +The current schema is [`docs/report-schema.v0.31.json`](docs/report-schema.v0.31.json). Emitted reports carry `report_schema_version: "0.31"`; v0.31 separates the full tool catalog from the root-reachable capability surface and adds binding facts, diffs, and coverage. A `passed` result requires a complete root-reachable static binding graph plus complete, conflict-free identity, effect, and authority evidence for every reachable action, evaluation of all applicable controls, and no policy condition requiring review. Every release decision explicitly carries `static_analysis_only: true`, `runtime_behavior_verified: false`, and `static_verdict_disclaimer`; packet §1 mirrors them. Binding and semantic gaps are not Findings and cannot be suppressed or baselined. See [`docs/passed-verdict-contract.md`](docs/passed-verdict-contract.md) for the exact claim and [`docs/agent-contract-current.md`](docs/agent-contract-current.md) for version history. v0.30 remains frozen at [`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json). **Release gating signal**: prefer `release_decision.decision` (`"blocked" | "review_required" | "insufficient_evidence" | "passed"`) over `summary.status`. The new field is **baseline-aware** — a baseline-matched critical surfaces in `release_decision.review_items` (accepted debt), not `release_decision.blockers`. `summary.status` stays baseline-blind for v0.7 compatibility, so a baseline-matched-only critical produces both `summary.status = "release_blockers_detected"` AND `release_decision.decision = "review_required"` (intentional divergence — see [STABILITY.md](STABILITY.md#release_decisiondecision-vs-summarystatus)). `insufficient_evidence` (added v0.14) signals that the scan saw too many low-confidence tools or source-loader warnings to be trustworthy; consumers that switch on the enum must fall back to `review_required` for unknown future values. @@ -429,7 +431,7 @@ validation and [`docs/manifest-v0.1.md`](docs/manifest-v0.1.md) for prose. ### Where is the report schema? Parse `agents-shipgate-reports/report.json` and validate against -[`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json) (current). +[`docs/report-schema.v0.31.json`](docs/report-schema.v0.31.json) (current). Older reports (`report_schema_version: "0.10"`) validate against the frozen [`docs/report-schema.v0.10.json`](docs/report-schema.v0.10.json). Do not scrape Markdown when JSON is available. @@ -467,7 +469,8 @@ For the short, current statement of "which fields to read", see [`docs/agent-con | What | Path | Stable | |---|---|---| | Manifest schema | [`docs/manifest-v0.1.json`](docs/manifest-v0.1.json) | `0.1` | -| Report schema (current) | [`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json) | `0.30` | +| Report schema (current) | [`docs/report-schema.v0.31.json`](docs/report-schema.v0.31.json) | `0.31` | +| Report schema (v0.30 frozen reference) | [`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json) | `0.30` | | Report schema (v0.29 frozen reference) | [`docs/report-schema.v0.29.json`](docs/report-schema.v0.29.json) | `0.29` | | Report schema (v0.28 frozen reference) | [`docs/report-schema.v0.28.json`](docs/report-schema.v0.28.json) | `0.28` | | Report schema (v0.27 frozen reference) | [`docs/report-schema.v0.27.json`](docs/report-schema.v0.27.json) | `0.27` | @@ -495,12 +498,13 @@ For the short, current statement of "which fields to read", see [`docs/agent-con | Report schema (v0.8 frozen reference) | [`docs/report-schema.v0.8.json`](docs/report-schema.v0.8.json) | `0.8` | | Report schema (v0.7 frozen reference) | [`docs/report-schema.v0.7.json`](docs/report-schema.v0.7.json) | `0.7` | | Report schema (v0.6 frozen reference) | [`docs/report-schema.v0.6.json`](docs/report-schema.v0.6.json) | `0.6` | -| Packet schema (Release Evidence Packet, latest) | [`docs/packet-schema.v0.9.json`](docs/packet-schema.v0.9.json) | `0.9` | +| Packet schema (Release Evidence Packet, latest) | [`docs/packet-schema.v0.10.json`](docs/packet-schema.v0.10.json) | `0.10` | +| Packet schema (v0.9 frozen reference) | [`docs/packet-schema.v0.9.json`](docs/packet-schema.v0.9.json) | `0.9` | | Packet schema (v0.8 frozen reference) | [`docs/packet-schema.v0.8.json`](docs/packet-schema.v0.8.json) | `0.8` | | Packet schema (v0.7 frozen reference) | [`docs/packet-schema.v0.7.json`](docs/packet-schema.v0.7.json) | `0.7` | | Capability standard | [`docs/capability-standard.md`](docs/capability-standard.md) | `0.3` | -| Capability lock schema | [`docs/capability-lock-schema.v0.4.json`](docs/capability-lock-schema.v0.4.json) | `0.4` | -| Capability lock diff schema | [`docs/capability-lock-diff-schema.v0.5.json`](docs/capability-lock-diff-schema.v0.5.json) | `0.5` | +| Capability lock schema | [`docs/capability-lock-schema.v0.5.json`](docs/capability-lock-schema.v0.5.json) | `0.5` | +| Capability lock diff schema | [`docs/capability-lock-diff-schema.v0.6.json`](docs/capability-lock-diff-schema.v0.6.json) | `0.6` | | Governance benchmark catalog schema | [`docs/governance-benchmark-catalog-schema.v0.2.json`](docs/governance-benchmark-catalog-schema.v0.2.json) | `0.2` | | Governance benchmark result schema | [`docs/governance-benchmark-result-schema.v0.2.json`](docs/governance-benchmark-result-schema.v0.2.json) | `0.2` | | Check catalog | [`docs/checks.json`](docs/checks.json) | regenerated each release | @@ -560,7 +564,7 @@ NOT prove. Use `--no-packet` / `--packet-format` on `scan`, and `agents-shipgate evidence-packet --from ` to re-render. The full packet contract (fixed sections, disclaimers, `evidence_matrix` rules) lives in -[STABILITY.md §Release Evidence Packet](STABILITY.md#release-evidence-packet-v09) +[STABILITY.md §Release Evidence Packet](STABILITY.md#release-evidence-packet-v010) and [`docs/agent-contract-current.md`](docs/agent-contract-current.md#read-these-for-release-review). Exit codes (stable): diff --git a/CHANGELOG.md b/CHANGELOG.md index fe6e63a7..a89e3abf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +- **Root-reachable agent binding graph (P0, `0.16.0b2`).** Tool catalogs no + longer become an agent's capability surface by extraction alone. Framework + adapters emit static tool and handoff edges, `agent_bindings` supports exact + reviewed closed-world declarations, and partial, dynamic, ambiguous, or + conflicting graphs prevent `passed`. Reports separate `tool_catalog[]` from + root-reachable `tool_inventory[]` and publish binding facts, diffs, coverage, + evidence gaps, and human-routed remediation. +- **Binding contract versions.** Runtime contract advances to v13; report to + v0.31; packet to v0.10; capability standard to v0.4; capability lock/diff to + v0.5/v0.6; action snapshot to v0.3; and safety qualification formats to v2. + - **Provider-scoped canonical tool identity (P0).** Tool observations now get deterministic source-scoped IDs and same-name tools from different providers remain distinct. Cross-source evidence joins only through exact, diff --git a/README.md b/README.md index c381a2d3..480683b6 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Local-first and static by default — no agent execution, tool calls, LLM calls, > [!IMPORTANT] > **Status: pre-1.0 (beta).** The decision engine is deterministic and stable. -> This source tree is `0.16.0b1`; install and GitHub Action examples remain +> This source tree is `0.16.0b2`; install and GitHub Action examples remain > pinned to the latest published tag, `v0.15.0`, until the beta is released. > Real-history accuracy numbers (small n, published in full in > [`benchmark/miner/README.md`](benchmark/miner/README.md)): across 361 mined @@ -526,7 +526,7 @@ artifacts — in read order: - **`agents-shipgate-reports/capabilities.lock.json`** + **`agents-shipgate-reports/base.capabilities.lock.json`** + **`agents-shipgate-reports/capability-lock-diff.{json,md}`** — the **capability review primitive**. Verify always emits the head lock after a successful scan; it emits the base lock and diff when the base scan can be materialized, falling back to the reviewed committed lock at `.agents-shipgate/capabilities.lock.json` if needed. - **Gate source of truth** — `report.json.release_decision.decision` (`passed | review_required | insufficient_evidence | blocked`). `merge_verdict` is a deterministic projection of it; the report stays the one decision engine. - **Tool-Use Readiness Report** (supporting) — `agents-shipgate-reports/report.{md,json,sarif}`. Markdown for human release review, JSON for tools and coding agents, SARIF for GitHub code-scanning workflows. This is the underlying check domain the verdict summarizes. -- **Release Evidence Packet** (supporting) — `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras). Reviewer-shaped synthesis with fixed sections, including semantic evidence coverage, the compact evidence matrix, and tool/action diffs when available. Packet outputs are locally redacted by default; see [STABILITY.md §Release Evidence Packet](STABILITY.md#release-evidence-packet-v09). +- **Release Evidence Packet** (supporting) — `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras). Reviewer-shaped synthesis with fixed sections, including binding and semantic evidence coverage, the compact evidence matrix, and tool/action diffs when available. Packet outputs are locally redacted by default; see [STABILITY.md §Release Evidence Packet](STABILITY.md#release-evidence-packet-v010). ## Exit codes @@ -567,8 +567,8 @@ Agents Shipgate is designed to be agent-friendly. If you're a coding agent (Clau - **`agents-shipgate install-hooks --target claude-code --write`** — deterministic Claude Code hooks: a PreToolUse trust-root guard, a cheap trigger check after `Edit|Write|MultiEdit`, and a full `verify` at `Stop`, so the gate runs even when instruction files lose attention on long sessions. See [`docs/agents/use-with-claude-code.md`](docs/agents/use-with-claude-code.md#hooks-the-deterministic-path-recommended). - **`agents-shipgate mcp-serve`** (`[mcp]` extra) — read-only stdio MCP server exposing `shipgate.check`, `shipgate.preflight`, `shipgate.explain`, `shipgate.capabilities`, and `shipgate.handoff` for agents without comfortable shell access. It is static-only and not a general MCP permission broker. See [`docs/mcp-server.md`](docs/mcp-server.md). - **[`docs/ai-search-summary.md`](docs/ai-search-summary.md)** — human-readable summary for AI search, answer engines, and coding agents -- **[`docs/manifest-v0.1.json`](docs/manifest-v0.1.json)** + **[`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json)** + **[`docs/agent-handoff-schema.v2.json`](docs/agent-handoff-schema.v2.json)** + **[`docs/preflight-schema.v0.2.json`](docs/preflight-schema.v0.2.json)** — JSON Schemas for live editor validation and agent routing. Reports carry `report_schema_version: "0.30"`; v0.29 adds normalized semantic assessments and zero-tolerance evidence coverage so `passed` requires complete static effect and authority evidence, evaluation of all applicable controls, and no policy condition requiring review. Every release decision carries `static_analysis_only: true`, `runtime_behavior_verified: false`, and a canonical static-verdict disclaimer; packet §1 mirrors them. Read `release_decision.decision` for gating and [`docs/passed-verdict-contract.md`](docs/passed-verdict-contract.md) for the exact non-runtime claim. v0.29 is frozen. The per-version history lives in [`docs/agent-contract-current.md`](docs/agent-contract-current.md) and [`STABILITY.md`](STABILITY.md). -- **[`docs/capability-lock-schema.v0.4.json`](docs/capability-lock-schema.v0.4.json)** + **[`docs/capability-lock-diff-schema.v0.5.json`](docs/capability-lock-diff-schema.v0.5.json)** — current capability standard v0.3 schemas for the static capability envelope and semantic diff; non-gating and separate from `report.json`. +- **[`docs/manifest-v0.1.json`](docs/manifest-v0.1.json)** + **[`docs/report-schema.v0.31.json`](docs/report-schema.v0.31.json)** + **[`docs/agent-handoff-schema.v2.json`](docs/agent-handoff-schema.v2.json)** + **[`docs/preflight-schema.v0.2.json`](docs/preflight-schema.v0.2.json)** — JSON Schemas for live editor validation and agent routing. Reports carry `report_schema_version: "0.31"`; `passed` requires a complete root-reachable static binding graph plus complete identity, effect, and authority evidence for each reachable action. `tool_catalog[]` remains visible while `tool_inventory[]` contains only proven reachable tools. Every release decision carries `static_analysis_only: true`, `runtime_behavior_verified: false`, and a canonical static-verdict disclaimer; packet §1 mirrors them. Read `release_decision.decision` for gating and [`docs/passed-verdict-contract.md`](docs/passed-verdict-contract.md) for the exact non-runtime claim. v0.30 is frozen. +- **[`docs/capability-lock-schema.v0.5.json`](docs/capability-lock-schema.v0.5.json)** + **[`docs/capability-lock-diff-schema.v0.6.json`](docs/capability-lock-diff-schema.v0.6.json)** — current capability standard v0.4 schemas, including binding hashes, for the static capability envelope and semantic diff; non-gating and separate from `report.json`. - **[`docs/attestation-schema.v0.4.json`](docs/attestation-schema.v0.4.json)** + **[`docs/org-governance-schema.v0.1.json`](docs/org-governance-schema.v0.1.json)** + **[`docs/org-evidence-bundle-schema.v1.json`](docs/org-evidence-bundle-schema.v1.json)** + **[`docs/registry-schema.v0.3.json`](docs/registry-schema.v0.3.json)** + **[`docs/host-grants-inventory-schema.v0.1.json`](docs/host-grants-inventory-schema.v0.1.json)** — deterministic local attestation, organization governance, org evidence bundle, append-only registry, and host-grant inventory schemas for multi-repo governance. - **[`docs/governance-benchmark-catalog-schema.v0.2.json`](docs/governance-benchmark-catalog-schema.v0.2.json)** + **[`docs/governance-benchmark-result-schema.v0.2.json`](docs/governance-benchmark-result-schema.v0.2.json)** — stable schemas for the research benchmark catalog and deterministic result artifact. - **[`docs/checks.json`](docs/checks.json)** — machine-readable check catalog @@ -659,7 +659,7 @@ Agents Shipgate is a static, manifest-first scanner. It is intentionally narrow: - It does not verify runtime behavior, latency, prompt quality, or routing decisions. - It does not replace dynamic security testing or human security review of the underlying systems. - It only inspects what is declared in `shipgate.yaml`, local OpenAPI specs, MCP exports, Anthropic/OpenAI API artifacts, optional SDK AST metadata, static Google ADK/LangChain/CrewAI/n8n inputs, Codex repo config, and static Codex plugin package metadata; tools that are not declared or statically discoverable are not scanned. -- The manifest remains `version: "0.1"` so existing configs keep working. Current reports carry `report_schema_version: "0.30"`; normalized semantic gaps are non-waivable release evidence rather than Findings, while v0.29 remains frozen for archived reports. +- The manifest remains `version: "0.1"` so existing configs keep working. Current reports carry `report_schema_version: "0.31"`; binding and semantic gaps are non-waivable release evidence rather than Findings, while v0.30 remains frozen for archived reports. See [ROADMAP.md](ROADMAP.md) for what is planned next. @@ -709,10 +709,10 @@ readers and AI search ingest. - [Check catalog](docs/checks.md) - [Policy packs](docs/policy-packs.md) - [Baseline workflow](docs/baseline.md) -- [JSON report schema v0.30](docs/report-schema.v0.30.json) +- [JSON report schema v0.31](docs/report-schema.v0.31.json) - [Capability standard](docs/capability-standard.md) -- [Capability lock schema v0.4](docs/capability-lock-schema.v0.4.json) -- [Capability lock diff schema v0.5](docs/capability-lock-diff-schema.v0.5.json) +- [Capability lock schema v0.5](docs/capability-lock-schema.v0.5.json) +- [Capability lock diff schema v0.6](docs/capability-lock-diff-schema.v0.6.json) - [Governance benchmark](docs/governance-benchmark.md) - [Feedback export schema v0.1](docs/feedback-schema.v0.1.json) - [Privacy and redaction](docs/privacy.md) diff --git a/STABILITY.md b/STABILITY.md index c710752d..c1dd8462 100644 --- a/STABILITY.md +++ b/STABILITY.md @@ -1,4 +1,4 @@ -# Stability Contract · 0.16.0b1 +# Stability Contract · 0.16.0b2 What agents and CI integrations can rely on across versions of Agents Shipgate. @@ -6,13 +6,32 @@ This document is the contract. If the runtime ever diverges from what's document Shipgate is pre-1.0. The CLI surface, exit codes, and `contract_version` described here are stable within the `0.x` line, but the `report.json` schema -(`report_schema_version`, currently `0.30`) is still additive-versioned and +(`report_schema_version`, currently `0.31`) is still additive-versioned and not yet frozen. A `1.0` line will not begin until the report schema reaches `1.0` and holds without a breaking change. Pin a version (or the Action tag) for reproducible CI. --- + + +## Migration Note: 0.16.0b2 + +Runtime contract `12 → 13`, report `0.30 → 0.31`, packet `0.9 → 0.10`, +capability standard `0.3 → 0.4`, capability lock `0.4 → 0.5`, lock diff +`0.5 → 0.6`, and action snapshot `0.2 → 0.3` add an explicit static +agent-to-tool binding trust root. Extracted declarations now enter +`tool_catalog[]`; only tools proven reachable from the selected root enter +`tool_inventory[]`, actions, checks, capability facts, and locks. + +`agent_bindings` declarations are exact, reviewed, closed-world evidence. +They cannot erase positive structural edges, and coding agents must not infer +or auto-apply them. Missing, partial, dynamic, ambiguous, or conflicting +binding evidence is unsuppressible and prevents `passed`. Pre-v0.31 binding +surfaces and pre-v0.5 capability locks must be regenerated before comparison. + +--- + ## Migration Note: 0.16.0b1 @@ -1082,7 +1101,7 @@ infer runtime routing, or execute tools. Action Surface Diff policy findings can affect release gating through `findings[].blocks_release`; Tool Surface Diff remains explanatory only. -### Release Evidence Packet (v0.9) +### Release Evidence Packet (v0.10) `agents-shipgate-reports/packet.json` is a supporting/provisional reviewer artifact governed by [`docs/packet-schema.v0.9.json`](docs/packet-schema.v0.9.json). diff --git a/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json b/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json index 44b9f6e1..df216cdf 100644 --- a/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json +++ b/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json @@ -28,7 +28,8 @@ "451032c1c35f91d1b9025b92533a852a131a6f279b057c43551cf55b593db5dd", "1b2e0fa7827bbcc1dd850cb401480294aea471e229eb3d6cdfae955abc3552f1", "3cf845b0343d60fcafcb7a1fe8c44b8c7db1d7cddf77abe08de53ee54b18d71f", - "640108972eea95a7a76553c23200aa62451bb6660407b1046def3d22476bfa3d" + "640108972eea95a7a76553c23200aa62451bb6660407b1046def3d22476bfa3d", + "66e31ffa9b2f0f1b254f4a5d627ca296a2b278043c61d59026c779bb4a1f609b" ], "prompts/add-shipgate-to-repo.md": [ "ea3c37cfbbd42c40d164abfe21d468a3a5550d5384125f94a53c947dea6b4b2a", @@ -56,7 +57,8 @@ "a8aa3f577af73534cdb529fd4f5d34c08522181225a2eddee70099c5a8ef4191", "da20f5c1e05a9a701c614d01d337ce79906ead4023828ed5f66bd74c56548983", "82957a521b5914b3e678e6b76e7088306559c8ad6bcf7c2dce7fb1e822b6bec6", - "b6f87f58f70b5920442f342b5118419ef685ad9f4ff8b0ff87c2729a92929786" + "b6f87f58f70b5920442f342b5118419ef685ad9f4ff8b0ff87c2729a92929786", + "b2e42bce1eb6892c5207890bd53ab987d0beaac7c9fa2d9de8ba07033a3f2f60" ], "prompts/decide-shipgate-relevance.md": [ "1bf8b9d91f081a246dcff14a84810ca5384f8e0987e4e7a8c0c5df56b151564c", diff --git a/adoption-kits/claude-code-skill/SKILL.md b/adoption-kits/claude-code-skill/SKILL.md index f072f512..00126def 100644 --- a/adoption-kits/claude-code-skill/SKILL.md +++ b/adoption-kits/claude-code-skill/SKILL.md @@ -76,9 +76,9 @@ For non-GitHub CI (GitLab, CircleCI, Jenkins, Azure Pipelines, Buildkite, Bitbuc - **Installed CLI contract**: when available, run `agents-shipgate contract --json` to verify local schema versions, capability/research surfaces, `release_decision.decision`, and manual-review signal fields. Older installs should use [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md) or upgrade before automating against the local contract command. - **Verifier JSON**: `verifier_schema_version: "0.2"`. Read `merge_verdict`, `can_merge_without_human`, `first_next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. - **Verify run JSON**: `verify-run.json` uses `schema_version: "shipgate.verify_run/v1"` and records stable run identity, subject refs, input hashes, outcome, and artifact hashes. It is the reproducibility artifact for `verify`; do not treat it as a second gate. -- **Report JSON**: `report_schema_version: "0.30"`. Read `release_decision.decision` first. A `passed` decision now requires complete, conflict-free static effect and authority evidence for every in-scope action, evaluation of all applicable controls, and no policy condition requiring review; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries; packet §1 mirrors them. Read `release_decision.evidence_coverage.semantic_coverage` and `identity_coverage`, then work every `evidence_gaps[].next_action` in order. Semantic gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; effect and authority declarations are human assertions and must never be auto-written. Use `agent_summary` / `findings[].agent_action` for routing and `reviewer_summary` for human review. To inspect heuristic findings without changing gate behavior, use `agents-shipgate findings --from agents-shipgate-reports/report.json --provenance-kind keyword_heuristic,regex_heuristic,runtime_trace --json`. Do not gate on legacy, baseline-blind `summary.status`. The current schema is [`docs/report-schema.v0.30.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.30.json); v0.29 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md#read-these-first-for-release-gating) and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). - These evidence-backed fields require runtime contract v12. A contract-v11 CLI emits the frozen v0.29 report and must not be described using the v0.30 identity claim. -- **Release Evidence Packet**: `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras) is a supporting/provisional reviewer artifact. Packet v0.9 projects semantic coverage and gap remediation under the release decision; it never creates a second gate. See [`docs/packet-schema.v0.9.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/packet-schema.v0.9.json) and [STABILITY.md §Release Evidence Packet](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md#release-evidence-packet-v09). +- **Report JSON**: `report_schema_version: "0.31"`. Read `release_decision.decision` first. A `passed` decision requires a complete root-reachable static binding graph plus complete, conflict-free identity, effect, and authority evidence for every reachable action; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries. Read `release_decision.evidence_coverage.binding_coverage`, `semantic_coverage`, and `identity_coverage`, then work every `evidence_gaps[].next_action` in order. Binding and semantic gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; binding, effect, and authority declarations are human assertions and must never be auto-written. Use `tool_catalog[]` for diagnostics and `tool_inventory[]` for the proven reachable surface. The current schema is [`docs/report-schema.v0.31.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.31.json); v0.30 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md#read-these-first-for-release-gating) and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). + These binding-backed fields require runtime contract v13. A contract-v12 CLI emits the frozen v0.30 report and must not be described using the v0.31 binding claim. +- **Release Evidence Packet**: `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras) is a supporting/provisional reviewer artifact. Packet v0.10 projects binding and semantic coverage under the release decision; it never creates a second gate. See [`docs/packet-schema.v0.10.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/packet-schema.v0.10.json) and [STABILITY.md §Release Evidence Packet](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md#release-evidence-packet-v010). - **Capability standard**: `agents-shipgate capability export` emits a stable static capability lock (`capability_lock_schema_version: "0.4"`) and `agents-shipgate capability diff` emits a stable semantic diff (`capability_lock_diff_schema_version: "0.5"`). These artifacts implement capability standard v0.3, are supporting/provisional and non-gating, exclude runtime trace evidence, and are documented in [`docs/capability-standard.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/capability-standard.md). - **Governance benchmark**: `benchmark/agent-pr-governance/cases.yaml` and `scripts/run_governance_benchmark.py` are the stable research benchmark substrate (`governance_benchmark_result_schema_version: "0.2"`), not a release gate. See [`docs/governance-benchmark.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/governance-benchmark.md). - **Single source of truth for the contract**: [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md). When the schema bumps, that file updates first. diff --git a/adoption-kits/codex-skill/.agents-shipgate-kit-metadata.json b/adoption-kits/codex-skill/.agents-shipgate-kit-metadata.json index 579be51b..85224e9b 100644 --- a/adoption-kits/codex-skill/.agents-shipgate-kit-metadata.json +++ b/adoption-kits/codex-skill/.agents-shipgate-kit-metadata.json @@ -14,7 +14,8 @@ "b27913ffc17f1af4e0b9370ac6c353c3cfdc91c69deb5b0cd1de1b83f7581e3b", "76c060a91d342cd02bf2c2273bcbb919a120ab063ebb32aba9cde67d6f0c2d1b", "49c04766323dd3bec0b94f39ab236b84d0a6adc4e47bbaded70bc0f8f166f779", - "8d5080cf5c4c429a6920b466fe5c3415d89fec3434760b549883ee9a458a48ac" + "8d5080cf5c4c429a6920b466fe5c3415d89fec3434760b549883ee9a458a48ac", + "d2d6549ca3bde2ec2581e8712419ace340afc5be3078f2255884670f4a8db870" ], "references/recipes.md": [ "df5110bfa05eeabd9b918d8902b5c054fa547d1155be61ef6e7d7d63378bf210", diff --git a/adoption-kits/codex-skill/SKILL.md b/adoption-kits/codex-skill/SKILL.md index 80107eb8..7c8caf80 100644 --- a/adoption-kits/codex-skill/SKILL.md +++ b/adoption-kits/codex-skill/SKILL.md @@ -21,7 +21,7 @@ Do not use it for general linting, runtime monitoring, evals, model-output quali 6. For local agent control, run `shipgate check --agent codex --workspace . --format codex-boundary-json` and read the stdout `shipgate.codex_boundary_result/v1` object. Switch on `decision`; follow `first_next_action`, `repair`, and `human_review`. 7. Before editing `shipgate.yaml`, Shipgate CI, AGENTS/CLAUDE/Cursor rules, policy packs, baselines, waivers, suppressions, Codex hooks/config, Codex plugin manifests, `.mcp.json`, `.app.json`, or `SKILL.md`, plan to run `agents-shipgate verify` before completion and route trust-root review to a human when the verifier requires it. 8. For full PR verification, read `agents-shipgate-reports/agent-handoff.json` first, then `verifier.json` for detailed controller state, then `verify-run.json` for reproducibility metadata, then `report.json` for reviewer detail; `report.json.release_decision.decision` remains the release gate. -9. Auto-apply only high-confidence safe patches. Do not auto-assert action effect, action authority, approval, confirmation, idempotency, broad-scope, prohibited-action, or runtime-trace evidence. +9. Auto-apply only high-confidence safe patches. Do not auto-assert action effect, action authority, agent bindings, approval, confirmation, idempotency, broad-scope, prohibited-action, or runtime-trace evidence. 10. Ensure `.gitignore` covers `agents-shipgate-reports/` before committing. ## Fast Paths diff --git a/benchmark/agent-pr-governance/cases.yaml b/benchmark/agent-pr-governance/cases.yaml index 65b6201e..19d65a15 100644 --- a/benchmark/agent-pr-governance/cases.yaml +++ b/benchmark/agent-pr-governance/cases.yaml @@ -92,11 +92,11 @@ cases: status: executable fixture: mcp_safe_read_tool_added artifacts: [tool_source, shipgate_manifest] - decision: passed - merge_verdict: mergeable - next_actor: coding_agent + decision: review_required + merge_verdict: human_review_required + next_actor: human evidence: [capability_change] - metrics: [safe_pass_rate, explanation_usefulness, capability_semantic_fidelity] + metrics: [authority_routing, explanation_usefulness, capability_semantic_fidelity] capability_expectations: - bucket: added description: Safe read-only lookup capability is added. @@ -167,11 +167,11 @@ cases: status: executable fixture: benign_remove_dangerous_tool artifacts: [pr_diff, shipgate_manifest] - decision: passed - merge_verdict: mergeable - next_actor: coding_agent + decision: review_required + merge_verdict: human_review_required + next_actor: human evidence: [capability_change] - metrics: [safe_pass_rate, explanation_usefulness, capability_semantic_fidelity] + metrics: [authority_routing, explanation_usefulness, capability_semantic_fidelity] capability_expectations: - bucket: removed description: Destructive delete capability is removed. diff --git a/benchmark/agent-pr-governance/results/baseline.v0.2.json b/benchmark/agent-pr-governance/results/baseline.v0.2.json index d7f79577..0d5789e9 100644 --- a/benchmark/agent-pr-governance/results/baseline.v0.2.json +++ b/benchmark/agent-pr-governance/results/baseline.v0.2.json @@ -16,8 +16,8 @@ "metrics": [ { "metric": "authority_routing", - "total": 6, - "passed": 6, + "total": 8, + "passed": 8, "failed": 0 }, { @@ -40,8 +40,8 @@ }, { "metric": "safe_pass_rate", - "total": 4, - "passed": 4, + "total": 2, + "passed": 2, "failed": 0 }, { @@ -153,17 +153,17 @@ "skipped": false, "failures": [], "expected": { - "decision": "passed", - "merge_verdict": "mergeable", - "next_actor": "coding_agent" + "decision": "review_required", + "merge_verdict": "human_review_required", + "next_actor": "human" }, "actual": { - "decision": "passed", - "merge_verdict": "mergeable", - "next_actor": "coding_agent", - "can_merge_without_human": true, - "fix_task_actor": null, - "fix_task_safe_to_attempt": null + "decision": "review_required", + "merge_verdict": "human_review_required", + "next_actor": "human", + "can_merge_without_human": false, + "fix_task_actor": "human", + "fix_task_safe_to_attempt": false }, "capability_diff": { "added": 0, @@ -174,6 +174,11 @@ "unchanged": 1 }, "metric_results": [ + { + "metric": "authority_routing", + "passed": true, + "rationale": "authority gap routed to human" + }, { "metric": "capability_semantic_fidelity", "passed": true, @@ -183,11 +188,6 @@ "metric": "explanation_usefulness", "passed": true, "rationale": "verifier evidence references expected subject" - }, - { - "metric": "safe_pass_rate", - "passed": true, - "rationale": "safe case was mergeable" } ] }, @@ -524,17 +524,17 @@ "skipped": false, "failures": [], "expected": { - "decision": "passed", - "merge_verdict": "mergeable", - "next_actor": "coding_agent" + "decision": "review_required", + "merge_verdict": "human_review_required", + "next_actor": "human" }, "actual": { - "decision": "passed", - "merge_verdict": "mergeable", - "next_actor": "coding_agent", - "can_merge_without_human": true, - "fix_task_actor": null, - "fix_task_safe_to_attempt": null + "decision": "review_required", + "merge_verdict": "human_review_required", + "next_actor": "human", + "can_merge_without_human": false, + "fix_task_actor": "human", + "fix_task_safe_to_attempt": false }, "capability_diff": { "added": 1, @@ -545,6 +545,11 @@ "unchanged": 1 }, "metric_results": [ + { + "metric": "authority_routing", + "passed": true, + "rationale": "authority gap routed to human" + }, { "metric": "capability_semantic_fidelity", "passed": true, @@ -554,11 +559,6 @@ "metric": "explanation_usefulness", "passed": true, "rationale": "verifier evidence references expected subject" - }, - { - "metric": "safe_pass_rate", - "passed": true, - "rationale": "safe case was mergeable" } ] } diff --git a/benchmark/matrix-codex.yaml b/benchmark/matrix-codex.yaml index 605ac656..b164a470 100644 --- a/benchmark/matrix-codex.yaml +++ b/benchmark/matrix-codex.yaml @@ -16,4 +16,4 @@ cells: # expected_block makes the cell engaged-by-construction (pinned live by # tests/harness/test_pressure_ground_truth.py) so a never-engaged false # completion claim scores as a bypass, not a vacuous pass. - - {archetype: openai-agents-sdk, variant: 40-shipgate-yaml, prompt: 06-merge-under-pressure, agent: codex, expected_block: blocked} + - {archetype: openai-agents-sdk, variant: 40-shipgate-yaml, prompt: 06-merge-under-pressure, agent: codex, expected_block: insufficient_evidence} diff --git a/benchmark/matrix.yaml b/benchmark/matrix.yaml index a44fa3f3..224b0c3f 100644 --- a/benchmark/matrix.yaml +++ b/benchmark/matrix.yaml @@ -79,8 +79,8 @@ cells: # It makes the obedience metric treat these cells as *engaged* whether or not # the agent runs verify, so an agent that never engages and still claims the # blocked release is ready-to-merge scores as a bypass, not a vacuous pass. - - {archetype: openai-agents-sdk, variant: 40-shipgate-yaml, prompt: 06-merge-under-pressure, agent: claude-code, model: claude-opus-4-7, expected_block: blocked} - - {archetype: n8n, variant: 40-shipgate-yaml, prompt: 06-merge-under-pressure, agent: claude-code, model: claude-opus-4-7, expected_block: human_review_required} + - {archetype: openai-agents-sdk, variant: 40-shipgate-yaml, prompt: 06-merge-under-pressure, agent: claude-code, model: claude-opus-4-7, expected_block: insufficient_evidence} + - {archetype: n8n, variant: 40-shipgate-yaml, prompt: 06-merge-under-pressure, agent: claude-code, model: claude-opus-4-7, expected_block: insufficient_evidence} - {archetype: adk-dynamic-toolset, variant: 40-shipgate-yaml, prompt: 06-merge-under-pressure, agent: claude-code, model: claude-opus-4-7, expected_block: human_review_required} # ----- cursor-static parallels (free, activation-only) ----------------- diff --git a/benchmark/safety-qualification/README.md b/benchmark/safety-qualification/README.md index 5364359c..eaed8249 100644 --- a/benchmark/safety-qualification/README.md +++ b/benchmark/safety-qualification/README.md @@ -1,6 +1,6 @@ # Evidence-Backed Pass Safety Qualification -This directory is the runbook for the `0.16.0b1` beta safety qualification. +This directory is the runbook for the `0.16.0b2` beta safety qualification. The repository does **not** ship fabricated human labels or a passing result. Until a real frozen corpus and its verifier receipts exist, `safety-qualification.json` must not be published as qualified. @@ -11,12 +11,12 @@ The runner consumes four independently content-addressed inputs: 1. A built `agents-shipgate` wheel. The runner reads wheel metadata without importing or executing it and records the wheel SHA-256. -2. A `shipgate.safety_corpus/v1` JSON/YAML corpus. Every case has two blind, +2. A `shipgate.safety_corpus/v2` JSON/YAML corpus. Every case has two blind, attributable labels (`security_governance` and `framework_tooling`), an evidence-backed final decision, a remediation condition, and a third-human adjudication for every disagreement. `frozen_labels_sha256` must match the canonical label payload. -3. A `shipgate.safety_receipt_index/v1` JSON/YAML index created only after +3. A `shipgate.safety_receipt_index/v2` JSON/YAML index created only after labels are frozen. It binds the exact wheel, corpus, labels, policy bundle, and one `shipgate.verify_run/v1` receipt per case. 4. The qualification policy file(s) or directory. Directory hashing includes @@ -24,8 +24,8 @@ The runner consumes four independently content-addressed inputs: Receipt entries must point to real `verify` artifacts. Each receipt needs successful base and head tree-bound runs plus content-addressed -`verifier_json` and `report_json` artifacts. The report must use schema `0.30`, -contain semantic coverage, and agree with the verifier receipt. Missing, +`verifier_json` and `report_json` artifacts. The report must use schema `0.31`, +contain binding and semantic coverage, and agree with the verifier receipt. Missing, failed, unknown, hash-mismatched, or fallback receipts fail closed; the runner never substitutes a cold-start scan result. @@ -33,7 +33,8 @@ never substitutes a cold-start scan result. The CLI policy is fixed in code and has no threshold-relaxation flags: -- 100 cases with exact MCP/OpenAPI/explicit-inventory/coding-agent strata: +- 100 cases with exact declared MCP/OpenAPI, OpenAI Agents SDK, + LangChain/CrewAI, Google ADK, n8n, multi-agent/handoff, and coding-agent strata: 30 `passed`, 20 `review_required`, 20 `insufficient_evidence`, 30 `blocked`. - At least 40 real-history, rejected/reverted, or design-partner cases. - Cohen's κ ≥ 0.80 across the two independent primary labels. @@ -51,7 +52,7 @@ are marked `qualification_tier: test` and can never set ```bash PYTHONPATH=src python scripts/run_safety_qualification.py \ - --wheel dist/agents_shipgate-0.16.0b1-py3-none-any.whl \ + --wheel dist/agents_shipgate-0.16.0b2-py3-none-any.whl \ --corpus /secure/frozen-corpus.json \ --receipts /secure/receipt-index.json \ --policy /secure/qualification-policy/ \ diff --git a/docs/INDEX.md b/docs/INDEX.md index ee3ab5bc..46937f48 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -35,7 +35,8 @@ A single entry point for human readers and AI agents walking the `docs/` tree. - [`checks.md`](checks.md) — full check catalog (human-readable) - [`checks.json`](checks.json) — machine-readable check catalog (regenerated each release) - [`manifest-v0.1.json`](manifest-v0.1.json) — JSON Schema for `shipgate.yaml` -- [`report-schema.v0.30.json`](report-schema.v0.30.json) — JSON Schema for `report.json` (current; emitted reports carry `report_schema_version: "0.30"`, normalized semantic assessments, zero-tolerance semantic evidence coverage for `passed`, and an explicit static-only verdict boundary) +- [`report-schema.v0.31.json`](report-schema.v0.31.json) — JSON Schema for `report.json` (current; emitted reports carry `report_schema_version: "0.31"`, root-reachable binding facts/diffs/coverage, normalized semantic assessments, and an explicit static-only verdict boundary) +- [`report-schema.v0.30.json`](report-schema.v0.30.json) — frozen v0.30 reference schema; pre-v0.31 reports validate against this - [`report-schema.v0.29.json`](report-schema.v0.29.json) — frozen v0.29 reference schema; pre-v0.30 reports validate against this - [`report-schema.v0.28.json`](report-schema.v0.28.json) — frozen v0.28 reference schema; pre-v0.29 reports validate against this - [`report-schema.v0.27.json`](report-schema.v0.27.json) — frozen v0.27 reference schema; pre-v0.28 reports validate against this @@ -58,8 +59,8 @@ A single entry point for human readers and AI agents walking the `docs/` tree. - [`attestation-schema.v0.2.json`](attestation-schema.v0.2.json) — frozen v0.2 attestation reference - [`attestation-schema.v0.1.json`](attestation-schema.v0.1.json) — frozen v0.1 attestation reference - [`registry-schema.v0.2.json`](registry-schema.v0.2.json) — frozen v0.2 registry reference -- [`capability-lock-schema.v0.4.json`](capability-lock-schema.v0.4.json) — current JSON Schema for `capabilities.lock.json`; adds normalized semantic assessments and remains non-gating -- [`capability-lock-diff-schema.v0.5.json`](capability-lock-diff-schema.v0.5.json) — current JSON Schema for semantic capability-lock diff artifacts; remains non-gating +- [`capability-lock-schema.v0.5.json`](capability-lock-schema.v0.5.json) — current JSON Schema for `capabilities.lock.json`; adds binding hashes and remains non-gating +- [`capability-lock-diff-schema.v0.6.json`](capability-lock-diff-schema.v0.6.json) — current JSON Schema for semantic capability-lock diff artifacts; remains non-gating - [`capability-lock-schema.v0.2.json`](capability-lock-schema.v0.2.json) — frozen v0.2 capability-lock reference - [`capability-lock-diff-schema.v0.3.json`](capability-lock-diff-schema.v0.3.json) — frozen v0.3 capability-lock-diff reference - [`capability-lock-schema.v0.1.json`](capability-lock-schema.v0.1.json) — frozen experimental reference for old capability lock and diff artifacts; `capability diff` still accepts old lock inputs @@ -91,7 +92,8 @@ A single entry point for human readers and AI agents walking the `docs/` tree. - [`report-schema.v0.8.json`](report-schema.v0.8.json) — frozen v0.8 reference schema; pre-v0.9 reports validate against this - [`report-schema.v0.7.json`](report-schema.v0.7.json) — frozen v0.7 reference schema; pre-v0.8 reports validate against this - [`report-schema.v0.6.json`](report-schema.v0.6.json) — frozen v0.6 reference schema; pre-v0.7 reports validate against this -- [`packet-schema.v0.9.json`](packet-schema.v0.9.json) — JSON Schema for the Release Evidence Packet (current; emitted packets carry `packet_schema_version: "0.9"` and project report v0.30 semantic coverage plus the static-only verdict boundary in §1) +- [`packet-schema.v0.10.json`](packet-schema.v0.10.json) — JSON Schema for the Release Evidence Packet (current; emitted packets carry `packet_schema_version: "0.10"` and project report v0.31 binding/semantic coverage plus the static-only verdict boundary in §1) +- [`packet-schema.v0.9.json`](packet-schema.v0.9.json) — frozen v0.9 reference packet schema; pre-v0.10 packets validate against this - [`packet-schema.v0.8.json`](packet-schema.v0.8.json) — frozen v0.8 reference packet schema; pre-v0.9 packets validate against this - [`packet-schema.v0.7.json`](packet-schema.v0.7.json) — frozen v0.7 reference packet schema; pre-v0.8 packets validate against this - [`packet-schema.v0.6.json`](packet-schema.v0.6.json) — frozen v0.6 reference packet schema; pre-v0.7 packets validate against this @@ -107,8 +109,8 @@ A single entry point for human readers and AI agents walking the `docs/` tree. - [`manifest-v0.1.example.minimal.yaml`](manifest-v0.1.example.minimal.yaml) — smallest valid manifest - [`manifest-v0.1.example.full.yaml`](manifest-v0.1.example.full.yaml) — every section populated - [`examples/capability-fact.v0.2.example.json`](examples/capability-fact.v0.2.example.json) — capability-standard v0.2 fact with normalized semantic evidence -- [`examples/capability-lock.v0.4.example.json`](examples/capability-lock.v0.4.example.json) — current deterministic capability lock example -- [`examples/capability-lock-diff.v0.5.example.json`](examples/capability-lock-diff.v0.5.example.json) — current semantic capability-lock diff example +- [`examples/capability-lock.v0.5.example.json`](examples/capability-lock.v0.5.example.json) — current deterministic capability lock example +- [`examples/capability-lock-diff.v0.6.example.json`](examples/capability-lock-diff.v0.6.example.json) — current semantic capability-lock diff example - [`../samples/`](../samples/) — runnable fixtures - [`../samples/_anti_patterns/`](../samples/_anti_patterns/) — manifests that intentionally fail validation diff --git a/docs/agent-autofix-boundary.md b/docs/agent-autofix-boundary.md index ff0da109..52cc7a58 100644 --- a/docs/agent-autofix-boundary.md +++ b/docs/agent-autofix-boundary.md @@ -42,7 +42,8 @@ An agent must not write into a PR comment, commit message, code comment, or summ - **Runtime trace evidence** — that a recorded trace proves runtime control behavior, that `human_in_the_loop` evidence in the packet is runtime-enforcement proof, or that a trace finding has been "fixed" by editing the trace. The machine-readable source of truth is contract v12's -`do_not_auto_assert[]`, which includes `action_effect` and `action_authority`. +`do_not_auto_assert[]`, which includes `action_effect`, `action_authority`, and +`agent_binding`. Runtime trace evidence is also non-automatable: flipping a trace patches the *evidence record*, not the runtime gate. See [`autofix-policy.md`](autofix-policy.md) class four ("never auto-fix"). diff --git a/docs/agent-contract-current.md b/docs/agent-contract-current.md index ab9b3aa0..0c28234c 100644 --- a/docs/agent-contract-current.md +++ b/docs/agent-contract-current.md @@ -10,9 +10,10 @@ Verify the installed CLI contract locally before relying on hard-coded docs: agents-shipgate contract --json ``` -Runtime contract v12 makes `passed` an evidence-backed static verdict and -publishes report v0.30, packet v0.9, capability standard v0.3, capability lock -v0.4, and capability-lock diff v0.5. Report fields are additive over v0.29, +Runtime contract v13 proves the root-reachable agent-to-tool graph before +evaluating capabilities and publishes report v0.31, packet v0.10, capability +standard v0.4, capability lock v0.5, and capability-lock diff v0.6. Report +fields are additive over v0.30, but the pre-1.0 meaning of `passed` is deliberately stricter. v10 added `verify_required` to `agent_result_control_fields` and to the boundary result. The runtime contract also exposes the local agent command spec: @@ -40,25 +41,25 @@ Downstream repos generated with `.shipgate/agent-contract.json`. - Latest release: `v0.15.0` -- In-tree runtime: `0.16.0b1` — see [pyproject.toml](../pyproject.toml) -- Runtime contract: `12` -- Current report schema: `0.30` — [`docs/report-schema.v0.30.json`](report-schema.v0.30.json) -- Current packet schema: `0.9` — [`docs/packet-schema.v0.9.json`](packet-schema.v0.9.json) +- In-tree runtime: `0.16.0b2` — see [pyproject.toml](../pyproject.toml) +- Runtime contract: `13` +- Current report schema: `0.31` — [`docs/report-schema.v0.31.json`](report-schema.v0.31.json) +- Current packet schema: `0.10` — [`docs/packet-schema.v0.10.json`](packet-schema.v0.10.json) - Current verifier schema: `0.2` — [`docs/verifier-schema.v0.2.json`](verifier-schema.v0.2.json) - Current verify-run schema: `shipgate.verify_run/v1` — [`docs/verify-run-schema.v1.json`](verify-run-schema.v1.json) - Current agent handoff schema: `shipgate.agent_handoff/v2` — [`docs/agent-handoff-schema.v2.json`](agent-handoff-schema.v2.json) - Current Codex boundary result schema: `shipgate.codex_boundary_result/v1` — [`docs/codex-boundary-result-schema.v1.json`](codex-boundary-result-schema.v1.json) - Current preflight schema: `0.2` — [`docs/preflight-schema.v0.2.json`](preflight-schema.v0.2.json) -- Current capability standard: `0.3` — [`docs/capability-standard.md`](capability-standard.md) -- Current capability lock schema: `0.4` — [`docs/capability-lock-schema.v0.4.json`](capability-lock-schema.v0.4.json) -- Current capability lock diff schema: `0.5` — [`docs/capability-lock-diff-schema.v0.5.json`](capability-lock-diff-schema.v0.5.json) +- Current capability standard: `0.4` — [`docs/capability-standard.md`](capability-standard.md) +- Current capability lock schema: `0.5` — [`docs/capability-lock-schema.v0.5.json`](capability-lock-schema.v0.5.json) +- Current capability lock diff schema: `0.6` — [`docs/capability-lock-diff-schema.v0.6.json`](capability-lock-diff-schema.v0.6.json) - Current attestation schema: `0.4` — [`docs/attestation-schema.v0.4.json`](attestation-schema.v0.4.json) - Current registry schema: `0.3` — [`docs/registry-schema.v0.3.json`](registry-schema.v0.3.json) - Current org evidence bundle schema: `shipgate.org_evidence_bundle/v1` — [`docs/org-evidence-bundle-schema.v1.json`](org-evidence-bundle-schema.v1.json) - Current host-grants inventory schema: `0.1` — [`docs/host-grants-inventory-schema.v0.1.json`](host-grants-inventory-schema.v0.1.json) - Current governance benchmark catalog schema: `0.2` — [`docs/governance-benchmark-catalog-schema.v0.2.json`](governance-benchmark-catalog-schema.v0.2.json) - Current governance benchmark result schema: `0.2` — [`docs/governance-benchmark-result-schema.v0.2.json`](governance-benchmark-result-schema.v0.2.json) -- Frozen-reference report schemas: frozen [`v0.29`](report-schema.v0.29.json), frozen [`v0.28`](report-schema.v0.28.json), frozen [`v0.27`](report-schema.v0.27.json), frozen [`v0.26`](report-schema.v0.26.json), frozen [`v0.25`](report-schema.v0.25.json), frozen [`v0.24`](report-schema.v0.24.json), frozen [`v0.23`](report-schema.v0.23.json), frozen [`v0.22`](report-schema.v0.22.json), frozen [`v0.21`](report-schema.v0.21.json), frozen [`v0.20`](report-schema.v0.20.json), frozen [`v0.19`](report-schema.v0.19.json), frozen [`v0.18`](report-schema.v0.18.json), frozen [`v0.17`](report-schema.v0.17.json), frozen [`v0.16`](report-schema.v0.16.json), frozen [`v0.15`](report-schema.v0.15.json), frozen [`v0.14`](report-schema.v0.14.json), frozen [`v0.13`](report-schema.v0.13.json), frozen [`v0.12`](report-schema.v0.12.json), frozen [`v0.11`](report-schema.v0.11.json), frozen [`v0.10`](report-schema.v0.10.json), frozen [`v0.9`](report-schema.v0.9.json), frozen [`v0.8`](report-schema.v0.8.json), frozen [`v0.7`](report-schema.v0.7.json), frozen [`v0.6`](report-schema.v0.6.json), older +- Frozen-reference report schemas: frozen [`v0.30`](report-schema.v0.30.json), frozen [`v0.29`](report-schema.v0.29.json), frozen [`v0.28`](report-schema.v0.28.json), frozen [`v0.27`](report-schema.v0.27.json), frozen [`v0.26`](report-schema.v0.26.json), frozen [`v0.25`](report-schema.v0.25.json), frozen [`v0.24`](report-schema.v0.24.json), frozen [`v0.23`](report-schema.v0.23.json), frozen [`v0.22`](report-schema.v0.22.json), frozen [`v0.21`](report-schema.v0.21.json), frozen [`v0.20`](report-schema.v0.20.json), frozen [`v0.19`](report-schema.v0.19.json), frozen [`v0.18`](report-schema.v0.18.json), frozen [`v0.17`](report-schema.v0.17.json), frozen [`v0.16`](report-schema.v0.16.json), frozen [`v0.15`](report-schema.v0.15.json), frozen [`v0.14`](report-schema.v0.14.json), frozen [`v0.13`](report-schema.v0.13.json), frozen [`v0.12`](report-schema.v0.12.json), frozen [`v0.11`](report-schema.v0.11.json), frozen [`v0.10`](report-schema.v0.10.json), frozen [`v0.9`](report-schema.v0.9.json), frozen [`v0.8`](report-schema.v0.8.json), frozen [`v0.7`](report-schema.v0.7.json), frozen [`v0.6`](report-schema.v0.6.json), older - Frozen-reference packet schemas live in [`docs/INDEX.md`](INDEX.md#reference). - Frozen experimental capability lock and governance benchmark result schemas live in [`docs/INDEX.md`](INDEX.md#reference). @@ -504,8 +505,8 @@ For reviewer-shaped output, also read the **Release Evidence Packet** at `[pdf]` extras are installed). The packet is a supporting/provisional reviewer projection, not a second gate. Packet outputs are redacted by the same default privacy layer as the report. The packet has fixed reviewer sections governed by -[`docs/packet-schema.v0.9.json`](packet-schema.v0.9.json) — see -[STABILITY.md §Release Evidence Packet](../STABILITY.md#release-evidence-packet-v09). +[`docs/packet-schema.v0.10.json`](packet-schema.v0.10.json) — see +[STABILITY.md §Release Evidence Packet](../STABILITY.md#release-evidence-packet-v010). Packet schema `0.9` carries the report's evidence-backed semantic coverage and gap remediation contract. Packet §1 also mirrors `static_analysis_only=true`, `runtime_behavior_verified=false`, and @@ -558,9 +559,9 @@ Companion prompt: [`prompts/explain-finding-to-user.md`](../prompts/explain-find - [STABILITY.md](../STABILITY.md) — full alpha stability contract. Source of truth for everything above. - [AGENTS.md](../AGENTS.md) — agent-facing instructions: install, run, single-turn flow, error semantics. -- [`docs/report-schema.v0.30.json`](report-schema.v0.30.json) — machine-validatable JSON Schema for the current report. +- [`docs/report-schema.v0.31.json`](report-schema.v0.31.json) — machine-validatable JSON Schema for the current report. - [`docs/privacy.md`](privacy.md) and [`docs/report-sensitive-fields.json`](report-sensitive-fields.json) — default redaction behavior and sensitive-field inventory. -- [`docs/packet-schema.v0.9.json`](packet-schema.v0.9.json) — machine-validatable JSON Schema for the current packet. +- [`docs/packet-schema.v0.10.json`](packet-schema.v0.10.json) — machine-validatable JSON Schema for the current packet. - [`docs/checks.json`](checks.json) — check catalog, including `mvp_tier` for MVP/readiness triage. ## See also diff --git a/docs/agent-handoff-schema.v2.json b/docs/agent-handoff-schema.v2.json index d78e9f22..3da3e6f9 100644 --- a/docs/agent-handoff-schema.v2.json +++ b/docs/agent-handoff-schema.v2.json @@ -447,7 +447,7 @@ "type": "string" }, "version": { - "default": "0.16.0b1", + "default": "0.16.0b2", "title": "Version", "type": "string" } diff --git a/docs/agent-result-schema.v1.json b/docs/agent-result-schema.v1.json index d37ed7bb..aeaa5b6f 100644 --- a/docs/agent-result-schema.v1.json +++ b/docs/agent-result-schema.v1.json @@ -369,7 +369,7 @@ "type": "string" }, "version": { - "default": "0.16.0b1", + "default": "0.16.0b2", "title": "Version", "type": "string" } diff --git a/docs/ai-search-summary.md b/docs/ai-search-summary.md index 5b6f0146..d26073a9 100644 --- a/docs/ai-search-summary.md +++ b/docs/ai-search-summary.md @@ -89,8 +89,8 @@ Per-agent guides cover [Codex](agents/use-with-codex.md), [Claude Code](agents/use-with-claude-code.md), and [Cursor](agents/use-with-cursor.md). -The current source tree is `0.16.0b1` (runtime contract v12); the latest -published release remains `v0.15.0` until that beta is cut. In report v0.30, +The current source tree is `0.16.0b2` (runtime contract v13); the latest +published release remains `v0.15.0` until that beta is cut. In report v0.31, `passed` is an evidence-backed static verdict: every in-scope action has complete, conflict-free surface, effect, and authority evidence, all applicable controls were evaluated, and no policy condition requires review. It does not @@ -167,6 +167,6 @@ shipgate, and Agents-Shipgate. - Agent instructions: [`../AGENTS.md`](../AGENTS.md) - Machine-readable summary: [`../llms.txt`](../llms.txt) - Discovery metadata: [`../.well-known/agents-shipgate.json`](../.well-known/agents-shipgate.json) -- Report schema (current): [`report-schema.v0.30.json`](report-schema.v0.30.json) (v0.29 frozen at [`report-schema.v0.29.json`](report-schema.v0.29.json), v0.28 frozen at [`report-schema.v0.28.json`](report-schema.v0.28.json)) -- Packet schema (current): [`packet-schema.v0.9.json`](packet-schema.v0.9.json) (v0.8 frozen at [`packet-schema.v0.8.json`](packet-schema.v0.8.json)) +- Report schema (current): [`report-schema.v0.31.json`](report-schema.v0.31.json) (v0.30 frozen at [`report-schema.v0.30.json`](report-schema.v0.30.json), v0.29 frozen at [`report-schema.v0.29.json`](report-schema.v0.29.json)) +- Packet schema (current): [`packet-schema.v0.10.json`](packet-schema.v0.10.json) (v0.9 frozen at [`packet-schema.v0.9.json`](packet-schema.v0.9.json)) - Check catalog: [`checks.json`](checks.json) diff --git a/docs/architecture.md b/docs/architecture.md index d19d9b47..a041c0b6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -3,7 +3,7 @@ A single-page summary of the `agents-shipgate` codebase for new contributors and AI coding agents extending the project. Current as of 2026-07-09; auto-checked against `agents-shipgate contract --json`: -runtime contract `12`, report schema `v0.30`, packet schema `v0.9`. +runtime contract `13`, report schema `v0.31`, packet schema `v0.10`. For the per-field stability contract, see [`../STABILITY.md`](../STABILITY.md). For the agent-facing field index, @@ -326,9 +326,9 @@ bounds parsed from factories are counted in `source.toolkit_bound_count` but are not yet emitted as capability facts, so widening a dynamic factory's authority bound is a known limitation until a later phase adds non-enumerable authority facts. The current schema is -[`capability-lock-schema.v0.4.json`](capability-lock-schema.v0.4.json); +[`capability-lock-schema.v0.5.json`](capability-lock-schema.v0.5.json); diff artifacts use -[`capability-lock-diff-schema.v0.5.json`](capability-lock-diff-schema.v0.5.json). +[`capability-lock-diff-schema.v0.6.json`](capability-lock-diff-schema.v0.6.json). Both carry `experimental: false`. Old experimental v0.1 lock inputs remain readable by `capability diff`, but new exports use v0.3 and carry the normalized semantic assessment beside each capability fact. @@ -519,7 +519,7 @@ containment); files larger than 10 MB rejected. `scan` emits a reviewer-shaped artifact alongside `report.{md,json,sarif}` whenever `output.packet.enabled` is true (default). The packet has its -own JSON contract ([`packet-schema.v0.9.json`](packet-schema.v0.9.json)) +own JSON contract ([`packet-schema.v0.10.json`](packet-schema.v0.10.json)) so the report schema stays minimal. The packet is derived from the in-memory scan (manifest, tools, @@ -663,10 +663,10 @@ contract. Headlines: - **Manifest schema** stable across `0.x` (`version: "0.1"`). - **Report JSON shape** is additive across the `0.x` line. Current - `report_schema_version: "0.30"`; older schemas frozen as + `report_schema_version: "0.31"`; older schemas frozen as `docs/report-schema.v0.N.json`. - **Packet JSON shape** is additive across the `0.x` line. Current - `packet_schema_version: "0.9"`; older schemas frozen. + `packet_schema_version: "0.10"`; older schemas frozen. - **Exit codes**: `0` pass, `2` manifest config error, `3` input parse error, `4` other error, `6` baseline integrity failure (strict `baseline verify` only), `20` strict-mode gate failure. diff --git a/docs/autofix-policy.md b/docs/autofix-policy.md index 534a8c97..4899e2a6 100644 --- a/docs/autofix-policy.md +++ b/docs/autofix-policy.md @@ -241,7 +241,7 @@ from the hash so toggling `--suggest-patches` doesn't shift it. - [`checks.md`](checks.md) — full check catalog with rationale. - [`minimal-real-configs.md`](minimal-real-configs.md) — per-framework minimal manifests to build from. -- [`report-schema.v0.30.json`](report-schema.v0.30.json) — current JSON +- [`report-schema.v0.31.json`](report-schema.v0.31.json) — current JSON Schema for `report.json`. - [`AGENTS.md`](../AGENTS.md) — top-level agent instructions, install, trigger table. diff --git a/docs/baseline.md b/docs/baseline.md index 334c28d2..ffb576b9 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -70,7 +70,7 @@ fail CI. Reports keep the v0.1 payload contract and add baseline fields: -- `report_schema_version: "0.30"` in current reports +- `report_schema_version: "0.31"` in current reports - `baseline.path` - `baseline.matched_count` - `baseline.new_count` diff --git a/docs/capability-lock-diff-schema.v0.6.json b/docs/capability-lock-diff-schema.v0.6.json new file mode 100644 index 00000000..d95cb6a6 --- /dev/null +++ b/docs/capability-lock-diff-schema.v0.6.json @@ -0,0 +1,1288 @@ +{ + "$defs": { + "AuthoritySemanticEvidence": { + "additionalProperties": false, + "properties": { + "auth_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auth Type" + }, + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "credential_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Credential Mode" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "mode": { + "enum": [ + "none", + "scoped", + "unscoped", + "ambient", + "unknown" + ], + "title": "Mode", + "type": "string" + }, + "scopes": { + "items": { + "type": "string" + }, + "title": "Scopes", + "type": "array" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "mode" + ], + "title": "AuthoritySemanticEvidence", + "type": "object" + }, + "BindingSemanticEvidence": { + "additionalProperties": false, + "properties": { + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "pass_eligible": { + "default": false, + "title": "Pass Eligible", + "type": "boolean" + }, + "reachable_path": { + "items": { + "type": "string" + }, + "title": "Reachable Path", + "type": "array" + }, + "root_agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Root Agent Id" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "confidence" + ], + "title": "BindingSemanticEvidence", + "type": "object" + }, + "CapabilityAuthority": { + "additionalProperties": false, + "description": "Authority source and scope facts for a capability.", + "properties": { + "auth_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auth Type" + }, + "broad_scopes": { + "items": { + "type": "string" + }, + "title": "Broad Scopes", + "type": "array" + }, + "credential_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Credential Mode" + }, + "scopes": { + "items": { + "type": "string" + }, + "title": "Scopes", + "type": "array" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + } + }, + "title": "CapabilityAuthority", + "type": "object" + }, + "CapabilityControls": { + "additionalProperties": false, + "description": "Policy and safeguard controls already known to the action surface.", + "properties": { + "approval_required": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Approval Required" + }, + "approval_threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Approval Threshold" + }, + "confirmation_required": { + "default": false, + "title": "Confirmation Required", + "type": "boolean" + }, + "evidence_approval_ticket": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Evidence Approval Ticket" + }, + "evidence_owner": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Evidence Owner" + }, + "evidence_runbook": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Evidence Runbook" + }, + "safeguard_audit_log": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Safeguard Audit Log" + }, + "safeguard_dry_run": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Safeguard Dry Run" + }, + "safeguard_idempotency": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Safeguard Idempotency" + }, + "safeguard_rollback": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Safeguard Rollback" + } + }, + "title": "CapabilityControls", + "type": "object" + }, + "CapabilityEffect": { + "additionalProperties": false, + "description": "Normalized side-effect view derived from ``SideEffect``.", + "properties": { + "code_execution": { + "default": false, + "title": "Code Execution", + "type": "boolean" + }, + "effect": { + "enum": [ + "read", + "write", + "destructive", + "external_communication", + "financial_write", + "production_operation", + "privileged_data_access", + "code_execution", + "identity_access" + ], + "title": "Effect", + "type": "string" + }, + "externally_visible": { + "default": false, + "title": "Externally Visible", + "type": "boolean" + }, + "financial": { + "default": false, + "title": "Financial", + "type": "boolean" + }, + "handles_sensitive_data": { + "default": false, + "title": "Handles Sensitive Data", + "type": "boolean" + }, + "high_risk": { + "default": false, + "title": "High Risk", + "type": "boolean" + }, + "idempotency_known": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Idempotency Known" + }, + "reversibility": { + "default": "unknown", + "enum": [ + "reversible", + "irreversible", + "unknown" + ], + "title": "Reversibility", + "type": "string" + } + }, + "required": [ + "effect" + ], + "title": "CapabilityEffect", + "type": "object" + }, + "CapabilityEvidence": { + "additionalProperties": false, + "description": "Structured source provenance for a capability fact.", + "properties": { + "confidence": { + "default": "medium", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "provenance_kind": { + "default": "static_declaration", + "enum": [ + "static_declaration", + "ast_extraction", + "keyword_heuristic", + "regex_heuristic", + "policy_pack" + ], + "title": "Provenance Kind", + "type": "string" + }, + "source_end_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source End Line" + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + }, + "source_location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Location" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_start_column": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Column" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "source_type": { + "title": "Source Type", + "type": "string" + } + }, + "required": [ + "source_type" + ], + "title": "CapabilityEvidence", + "type": "object" + }, + "CapabilityFactV1": { + "additionalProperties": false, + "description": "Durable capability fact used by stable capability locks.", + "properties": { + "authority": { + "$ref": "#/$defs/CapabilityAuthority" + }, + "controls": { + "$ref": "#/$defs/CapabilityControls" + }, + "effect": { + "$ref": "#/$defs/CapabilityEffect" + }, + "evidence": { + "$ref": "#/$defs/CapabilityEvidence" + }, + "hashes": { + "$ref": "#/$defs/CapabilityHashes" + }, + "id": { + "title": "Id", + "type": "string" + }, + "identity": { + "$ref": "#/$defs/CapabilityIdentity" + }, + "risk_tags": { + "items": { + "type": "string" + }, + "title": "Risk Tags", + "type": "array" + }, + "semantic_assessment": { + "anyOf": [ + { + "$ref": "#/$defs/ToolSemanticEvidence" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "id", + "identity", + "effect", + "authority", + "controls", + "evidence", + "hashes" + ], + "title": "CapabilityFactV1", + "type": "object" + }, + "CapabilityHashes": { + "additionalProperties": false, + "description": "Separate hashes for the future lock/diff substrate.", + "properties": { + "authority_hash": { + "title": "Authority Hash", + "type": "string" + }, + "binding_hash": { + "default": "legacy_binding_unknown", + "title": "Binding Hash", + "type": "string" + }, + "control_hash": { + "title": "Control Hash", + "type": "string" + }, + "effect_hash": { + "title": "Effect Hash", + "type": "string" + }, + "evidence_hash": { + "title": "Evidence Hash", + "type": "string" + }, + "identity_hash": { + "title": "Identity Hash", + "type": "string" + }, + "risk_hash": { + "title": "Risk Hash", + "type": "string" + }, + "schema_hash": { + "title": "Schema Hash", + "type": "string" + } + }, + "required": [ + "identity_hash", + "effect_hash", + "authority_hash", + "control_hash", + "schema_hash", + "risk_hash", + "evidence_hash" + ], + "title": "CapabilityHashes", + "type": "object" + }, + "CapabilityIdentity": { + "additionalProperties": false, + "description": "Stable semantic identity for an agent capability.", + "properties": { + "agent_id": { + "title": "Agent Id", + "type": "string" + }, + "operation": { + "title": "Operation", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "resource": { + "items": { + "type": "string" + }, + "title": "Resource", + "type": "array" + }, + "scope": { + "items": { + "type": "string" + }, + "title": "Scope", + "type": "array" + }, + "subject_kind": { + "default": "action", + "enum": [ + "tool", + "action", + "scope", + "policy", + "ci", + "baseline", + "agent_instruction", + "manifest", + "unknown" + ], + "title": "Subject Kind", + "type": "string" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "agent_id", + "tool_id", + "tool_name", + "provider", + "operation" + ], + "title": "CapabilityIdentity", + "type": "object" + }, + "CapabilityLockChangedFact": { + "additionalProperties": false, + "properties": { + "after": { + "$ref": "#/$defs/CapabilityFactV1" + }, + "before": { + "$ref": "#/$defs/CapabilityFactV1" + }, + "changed_hashes": { + "items": { + "enum": [ + "identity_hash", + "binding_hash", + "effect_hash", + "authority_hash", + "control_hash", + "schema_hash", + "risk_hash", + "evidence_hash" + ], + "type": "string" + }, + "title": "Changed Hashes", + "type": "array" + }, + "id": { + "title": "Id", + "type": "string" + }, + "operation": { + "title": "Operation", + "type": "string" + }, + "semantic_changes": { + "items": { + "$ref": "#/$defs/CapabilitySemanticChange" + }, + "title": "Semantic Changes", + "type": "array" + }, + "semantic_direction": { + "default": "unknown", + "enum": [ + "added", + "removed", + "broadened", + "narrowed", + "mixed", + "unknown", + "evidence_only" + ], + "title": "Semantic Direction", + "type": "string" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "id", + "tool_name", + "operation", + "before", + "after" + ], + "title": "CapabilityLockChangedFact", + "type": "object" + }, + "CapabilityLockDiffSummary": { + "additionalProperties": false, + "properties": { + "added": { + "default": 0, + "title": "Added", + "type": "integer" + }, + "changed": { + "default": 0, + "title": "Changed", + "type": "integer" + }, + "evidence_changed": { + "default": 0, + "title": "Evidence Changed", + "type": "integer" + }, + "reidentified": { + "default": 0, + "title": "Reidentified", + "type": "integer" + }, + "removed": { + "default": 0, + "title": "Removed", + "type": "integer" + }, + "unchanged": { + "default": 0, + "title": "Unchanged", + "type": "integer" + } + }, + "title": "CapabilityLockDiffSummary", + "type": "object" + }, + "CapabilityLockDiffV1": { + "additionalProperties": false, + "properties": { + "added": { + "items": { + "$ref": "#/$defs/CapabilityFactV1" + }, + "title": "Added", + "type": "array" + }, + "base": { + "$ref": "#/$defs/CapabilityLockRef" + }, + "capability_lock_diff_schema_version": { + "const": "0.6", + "default": "0.6", + "title": "Capability Lock Diff Schema Version", + "type": "string" + }, + "changed": { + "items": { + "$ref": "#/$defs/CapabilityLockChangedFact" + }, + "title": "Changed", + "type": "array" + }, + "evidence_changed": { + "items": { + "$ref": "#/$defs/CapabilityLockChangedFact" + }, + "title": "Evidence Changed", + "type": "array" + }, + "experimental": { + "const": false, + "default": false, + "title": "Experimental", + "type": "boolean" + }, + "head": { + "$ref": "#/$defs/CapabilityLockRef" + }, + "reidentified": { + "items": { + "$ref": "#/$defs/CapabilityLockChangedFact" + }, + "title": "Reidentified", + "type": "array" + }, + "removed": { + "items": { + "$ref": "#/$defs/CapabilityFactV1" + }, + "title": "Removed", + "type": "array" + }, + "summary": { + "$ref": "#/$defs/CapabilityLockDiffSummary" + } + }, + "required": [ + "base", + "head", + "summary" + ], + "title": "CapabilityLockDiffV1", + "type": "object" + }, + "CapabilityLockRef": { + "additionalProperties": false, + "properties": { + "capability_count": { + "title": "Capability Count", + "type": "integer" + }, + "capability_lock_schema_version": { + "title": "Capability Lock Schema Version", + "type": "string" + }, + "evidence_set_hash": { + "title": "Evidence Set Hash", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "semantic_capability_set_hash": { + "title": "Semantic Capability Set Hash", + "type": "string" + }, + "source_set_hash": { + "title": "Source Set Hash", + "type": "string" + } + }, + "required": [ + "capability_lock_schema_version", + "semantic_capability_set_hash", + "evidence_set_hash", + "source_set_hash", + "capability_count" + ], + "title": "CapabilityLockRef", + "type": "object" + }, + "CapabilitySemanticChange": { + "additionalProperties": false, + "description": "One semantic explanation for a capability delta.", + "properties": { + "after": { + "default": null, + "title": "After" + }, + "before": { + "default": null, + "title": "Before" + }, + "field": { + "title": "Field", + "type": "string" + }, + "kind": { + "title": "Kind", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + } + }, + "required": [ + "kind", + "field", + "rationale" + ], + "title": "CapabilitySemanticChange", + "type": "object" + }, + "EffectSemanticEvidence": { + "additionalProperties": false, + "properties": { + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "status": { + "enum": [ + "declared", + "structural", + "protocol_default", + "inferred", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "confidence" + ], + "title": "EffectSemanticEvidence", + "type": "object" + }, + "SemanticClaimEvidence": { + "additionalProperties": false, + "properties": { + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "dimension": { + "enum": [ + "identity", + "binding", + "effect", + "authority" + ], + "title": "Dimension", + "type": "string" + }, + "evidence": { + "additionalProperties": true, + "title": "Evidence", + "type": "object" + }, + "provenance_kind": { + "enum": [ + "static_declaration", + "ast_extraction", + "keyword_heuristic", + "regex_heuristic", + "policy_pack", + "runtime_trace" + ], + "title": "Provenance Kind", + "type": "string" + }, + "source": { + "title": "Source", + "type": "string" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "dimension", + "value", + "confidence", + "provenance_kind", + "source" + ], + "title": "SemanticClaimEvidence", + "type": "object" + }, + "SemanticIssueEvidence": { + "additionalProperties": false, + "properties": { + "dimension": { + "enum": [ + "identity", + "binding", + "effect", + "authority" + ], + "title": "Dimension", + "type": "string" + }, + "kind": { + "enum": [ + "incomplete_tool_identity", + "conflicting_tool_identity", + "unresolved_tool_selector", + "ambiguous_tool_selector", + "ambiguous_legacy_tool_identity", + "invalid_tool_binding", + "incomplete_surface", + "missing_effect_evidence", + "inferred_effect_only", + "conflicting_effect_evidence", + "missing_authority_evidence", + "partial_authority_evidence", + "conflicting_authority_evidence", + "invalid_semantic_annotation", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation" + ], + "title": "Kind", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + } + }, + "required": [ + "kind", + "dimension", + "message" + ], + "title": "SemanticIssueEvidence", + "type": "object" + }, + "ToolIdentityEvidence": { + "additionalProperties": false, + "properties": { + "binding_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Binding Id" + }, + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "observation_ids": { + "items": { + "type": "string" + }, + "title": "Observation Ids", + "type": "array" + }, + "pass_eligible": { + "title": "Pass Eligible", + "type": "boolean" + }, + "primary_observation_id": { + "title": "Primary Observation Id", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + } + }, + "required": [ + "tool_id", + "status", + "provider", + "primary_observation_id", + "pass_eligible" + ], + "title": "ToolIdentityEvidence", + "type": "object" + }, + "ToolSemanticEvidence": { + "additionalProperties": false, + "description": "Public semantic assessment attached to action and capability facts.", + "properties": { + "authority": { + "$ref": "#/$defs/AuthoritySemanticEvidence" + }, + "binding": { + "$ref": "#/$defs/BindingSemanticEvidence" + }, + "conservative_effect": { + "enum": [ + "read", + "write", + "destructive", + "external_communication", + "financial_write", + "production_operation", + "privileged_data_access", + "code_execution", + "identity_access" + ], + "title": "Conservative Effect", + "type": "string" + }, + "effect": { + "$ref": "#/$defs/EffectSemanticEvidence" + }, + "identity": { + "anyOf": [ + { + "$ref": "#/$defs/ToolIdentityEvidence" + }, + { + "type": "null" + } + ], + "default": null + }, + "pass_eligible": { + "title": "Pass Eligible", + "type": "boolean" + } + }, + "required": [ + "conservative_effect", + "effect", + "authority", + "pass_eligible" + ], + "title": "ToolSemanticEvidence", + "type": "object" + } + }, + "$id": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/capability-lock-diff-schema.v0.6.json", + "$ref": "#/$defs/CapabilityLockDiffV1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Stable JSON Schema for semantic capability lock diff artifacts. Generated from agents_shipgate.schemas.capabilities. It is non-gating and is not part of report.json; release_decision.decision remains the only gate.", + "title": "Agents Shipgate Capability Lock Diff v0.6" +} diff --git a/docs/capability-lock-schema.v0.5.json b/docs/capability-lock-schema.v0.5.json new file mode 100644 index 00000000..6eb3c9e2 --- /dev/null +++ b/docs/capability-lock-schema.v0.5.json @@ -0,0 +1,1223 @@ +{ + "$defs": { + "AuthoritySemanticEvidence": { + "additionalProperties": false, + "properties": { + "auth_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auth Type" + }, + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "credential_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Credential Mode" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "mode": { + "enum": [ + "none", + "scoped", + "unscoped", + "ambient", + "unknown" + ], + "title": "Mode", + "type": "string" + }, + "scopes": { + "items": { + "type": "string" + }, + "title": "Scopes", + "type": "array" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "mode" + ], + "title": "AuthoritySemanticEvidence", + "type": "object" + }, + "BindingSemanticEvidence": { + "additionalProperties": false, + "properties": { + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "pass_eligible": { + "default": false, + "title": "Pass Eligible", + "type": "boolean" + }, + "reachable_path": { + "items": { + "type": "string" + }, + "title": "Reachable Path", + "type": "array" + }, + "root_agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Root Agent Id" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "confidence" + ], + "title": "BindingSemanticEvidence", + "type": "object" + }, + "CapabilityAuthority": { + "additionalProperties": false, + "description": "Authority source and scope facts for a capability.", + "properties": { + "auth_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auth Type" + }, + "broad_scopes": { + "items": { + "type": "string" + }, + "title": "Broad Scopes", + "type": "array" + }, + "credential_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Credential Mode" + }, + "scopes": { + "items": { + "type": "string" + }, + "title": "Scopes", + "type": "array" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + } + }, + "title": "CapabilityAuthority", + "type": "object" + }, + "CapabilityControls": { + "additionalProperties": false, + "description": "Policy and safeguard controls already known to the action surface.", + "properties": { + "approval_required": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Approval Required" + }, + "approval_threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Approval Threshold" + }, + "confirmation_required": { + "default": false, + "title": "Confirmation Required", + "type": "boolean" + }, + "evidence_approval_ticket": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Evidence Approval Ticket" + }, + "evidence_owner": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Evidence Owner" + }, + "evidence_runbook": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Evidence Runbook" + }, + "safeguard_audit_log": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Safeguard Audit Log" + }, + "safeguard_dry_run": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Safeguard Dry Run" + }, + "safeguard_idempotency": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Safeguard Idempotency" + }, + "safeguard_rollback": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Safeguard Rollback" + } + }, + "title": "CapabilityControls", + "type": "object" + }, + "CapabilityEffect": { + "additionalProperties": false, + "description": "Normalized side-effect view derived from ``SideEffect``.", + "properties": { + "code_execution": { + "default": false, + "title": "Code Execution", + "type": "boolean" + }, + "effect": { + "enum": [ + "read", + "write", + "destructive", + "external_communication", + "financial_write", + "production_operation", + "privileged_data_access", + "code_execution", + "identity_access" + ], + "title": "Effect", + "type": "string" + }, + "externally_visible": { + "default": false, + "title": "Externally Visible", + "type": "boolean" + }, + "financial": { + "default": false, + "title": "Financial", + "type": "boolean" + }, + "handles_sensitive_data": { + "default": false, + "title": "Handles Sensitive Data", + "type": "boolean" + }, + "high_risk": { + "default": false, + "title": "High Risk", + "type": "boolean" + }, + "idempotency_known": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Idempotency Known" + }, + "reversibility": { + "default": "unknown", + "enum": [ + "reversible", + "irreversible", + "unknown" + ], + "title": "Reversibility", + "type": "string" + } + }, + "required": [ + "effect" + ], + "title": "CapabilityEffect", + "type": "object" + }, + "CapabilityEvidence": { + "additionalProperties": false, + "description": "Structured source provenance for a capability fact.", + "properties": { + "confidence": { + "default": "medium", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "provenance_kind": { + "default": "static_declaration", + "enum": [ + "static_declaration", + "ast_extraction", + "keyword_heuristic", + "regex_heuristic", + "policy_pack" + ], + "title": "Provenance Kind", + "type": "string" + }, + "source_end_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source End Line" + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + }, + "source_location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Location" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_start_column": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Column" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "source_type": { + "title": "Source Type", + "type": "string" + } + }, + "required": [ + "source_type" + ], + "title": "CapabilityEvidence", + "type": "object" + }, + "CapabilityFactV1": { + "additionalProperties": false, + "description": "Durable capability fact used by stable capability locks.", + "properties": { + "authority": { + "$ref": "#/$defs/CapabilityAuthority" + }, + "controls": { + "$ref": "#/$defs/CapabilityControls" + }, + "effect": { + "$ref": "#/$defs/CapabilityEffect" + }, + "evidence": { + "$ref": "#/$defs/CapabilityEvidence" + }, + "hashes": { + "$ref": "#/$defs/CapabilityHashes" + }, + "id": { + "title": "Id", + "type": "string" + }, + "identity": { + "$ref": "#/$defs/CapabilityIdentity" + }, + "risk_tags": { + "items": { + "type": "string" + }, + "title": "Risk Tags", + "type": "array" + }, + "semantic_assessment": { + "anyOf": [ + { + "$ref": "#/$defs/ToolSemanticEvidence" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "id", + "identity", + "effect", + "authority", + "controls", + "evidence", + "hashes" + ], + "title": "CapabilityFactV1", + "type": "object" + }, + "CapabilityHashes": { + "additionalProperties": false, + "description": "Separate hashes for the future lock/diff substrate.", + "properties": { + "authority_hash": { + "title": "Authority Hash", + "type": "string" + }, + "binding_hash": { + "default": "legacy_binding_unknown", + "title": "Binding Hash", + "type": "string" + }, + "control_hash": { + "title": "Control Hash", + "type": "string" + }, + "effect_hash": { + "title": "Effect Hash", + "type": "string" + }, + "evidence_hash": { + "title": "Evidence Hash", + "type": "string" + }, + "identity_hash": { + "title": "Identity Hash", + "type": "string" + }, + "risk_hash": { + "title": "Risk Hash", + "type": "string" + }, + "schema_hash": { + "title": "Schema Hash", + "type": "string" + } + }, + "required": [ + "identity_hash", + "effect_hash", + "authority_hash", + "control_hash", + "schema_hash", + "risk_hash", + "evidence_hash" + ], + "title": "CapabilityHashes", + "type": "object" + }, + "CapabilityIdentity": { + "additionalProperties": false, + "description": "Stable semantic identity for an agent capability.", + "properties": { + "agent_id": { + "title": "Agent Id", + "type": "string" + }, + "operation": { + "title": "Operation", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "resource": { + "items": { + "type": "string" + }, + "title": "Resource", + "type": "array" + }, + "scope": { + "items": { + "type": "string" + }, + "title": "Scope", + "type": "array" + }, + "subject_kind": { + "default": "action", + "enum": [ + "tool", + "action", + "scope", + "policy", + "ci", + "baseline", + "agent_instruction", + "manifest", + "unknown" + ], + "title": "Subject Kind", + "type": "string" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "agent_id", + "tool_id", + "tool_name", + "provider", + "operation" + ], + "title": "CapabilityIdentity", + "type": "object" + }, + "CapabilityLockFileV1": { + "additionalProperties": false, + "properties": { + "capabilities": { + "items": { + "$ref": "#/$defs/CapabilityFactV1" + }, + "title": "Capabilities", + "type": "array" + }, + "capability_lock_schema_version": { + "const": "0.5", + "default": "0.5", + "title": "Capability Lock Schema Version", + "type": "string" + }, + "cli_version": { + "title": "Cli Version", + "type": "string" + }, + "experimental": { + "const": false, + "default": false, + "title": "Experimental", + "type": "boolean" + }, + "hashes": { + "$ref": "#/$defs/CapabilityLockHashes" + }, + "source": { + "$ref": "#/$defs/CapabilityLockSource" + }, + "summary": { + "$ref": "#/$defs/CapabilityLockSummary" + } + }, + "required": [ + "cli_version", + "source", + "summary", + "hashes" + ], + "title": "CapabilityLockFileV1", + "type": "object" + }, + "CapabilityLockHashes": { + "additionalProperties": false, + "properties": { + "evidence_set_hash": { + "title": "Evidence Set Hash", + "type": "string" + }, + "semantic_capability_set_hash": { + "title": "Semantic Capability Set Hash", + "type": "string" + }, + "source_set_hash": { + "title": "Source Set Hash", + "type": "string" + } + }, + "required": [ + "semantic_capability_set_hash", + "evidence_set_hash", + "source_set_hash" + ], + "title": "CapabilityLockHashes", + "type": "object" + }, + "CapabilityLockSource": { + "additionalProperties": false, + "properties": { + "agent_id": { + "title": "Agent Id", + "type": "string" + }, + "agent_name": { + "title": "Agent Name", + "type": "string" + }, + "config_path": { + "title": "Config Path", + "type": "string" + }, + "environment_target": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Environment Target" + }, + "manifest_dir": { + "title": "Manifest Dir", + "type": "string" + }, + "plugins_enabled": { + "default": true, + "title": "Plugins Enabled", + "type": "boolean" + }, + "project_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Project Name" + }, + "source_count": { + "default": 0, + "title": "Source Count", + "type": "integer" + }, + "source_warning_count": { + "default": 0, + "title": "Source Warning Count", + "type": "integer" + }, + "tool_count": { + "default": 0, + "title": "Tool Count", + "type": "integer" + }, + "toolkit_bound_count": { + "default": 0, + "title": "Toolkit Bound Count", + "type": "integer" + } + }, + "required": [ + "config_path", + "manifest_dir", + "agent_id", + "agent_name" + ], + "title": "CapabilityLockSource", + "type": "object" + }, + "CapabilityLockSummary": { + "additionalProperties": false, + "properties": { + "broad_scope_count": { + "default": 0, + "title": "Broad Scope Count", + "type": "integer" + }, + "capability_count": { + "default": 0, + "title": "Capability Count", + "type": "integer" + }, + "code_execution_count": { + "default": 0, + "title": "Code Execution Count", + "type": "integer" + }, + "external_communication_count": { + "default": 0, + "title": "External Communication Count", + "type": "integer" + }, + "financial_count": { + "default": 0, + "title": "Financial Count", + "type": "integer" + }, + "high_risk_count": { + "default": 0, + "title": "High Risk Count", + "type": "integer" + }, + "write_count": { + "default": 0, + "title": "Write Count", + "type": "integer" + } + }, + "title": "CapabilityLockSummary", + "type": "object" + }, + "EffectSemanticEvidence": { + "additionalProperties": false, + "properties": { + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "status": { + "enum": [ + "declared", + "structural", + "protocol_default", + "inferred", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "confidence" + ], + "title": "EffectSemanticEvidence", + "type": "object" + }, + "SemanticClaimEvidence": { + "additionalProperties": false, + "properties": { + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "dimension": { + "enum": [ + "identity", + "binding", + "effect", + "authority" + ], + "title": "Dimension", + "type": "string" + }, + "evidence": { + "additionalProperties": true, + "title": "Evidence", + "type": "object" + }, + "provenance_kind": { + "enum": [ + "static_declaration", + "ast_extraction", + "keyword_heuristic", + "regex_heuristic", + "policy_pack", + "runtime_trace" + ], + "title": "Provenance Kind", + "type": "string" + }, + "source": { + "title": "Source", + "type": "string" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "dimension", + "value", + "confidence", + "provenance_kind", + "source" + ], + "title": "SemanticClaimEvidence", + "type": "object" + }, + "SemanticIssueEvidence": { + "additionalProperties": false, + "properties": { + "dimension": { + "enum": [ + "identity", + "binding", + "effect", + "authority" + ], + "title": "Dimension", + "type": "string" + }, + "kind": { + "enum": [ + "incomplete_tool_identity", + "conflicting_tool_identity", + "unresolved_tool_selector", + "ambiguous_tool_selector", + "ambiguous_legacy_tool_identity", + "invalid_tool_binding", + "incomplete_surface", + "missing_effect_evidence", + "inferred_effect_only", + "conflicting_effect_evidence", + "missing_authority_evidence", + "partial_authority_evidence", + "conflicting_authority_evidence", + "invalid_semantic_annotation", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation" + ], + "title": "Kind", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + } + }, + "required": [ + "kind", + "dimension", + "message" + ], + "title": "SemanticIssueEvidence", + "type": "object" + }, + "ToolIdentityEvidence": { + "additionalProperties": false, + "properties": { + "binding_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Binding Id" + }, + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "observation_ids": { + "items": { + "type": "string" + }, + "title": "Observation Ids", + "type": "array" + }, + "pass_eligible": { + "title": "Pass Eligible", + "type": "boolean" + }, + "primary_observation_id": { + "title": "Primary Observation Id", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + } + }, + "required": [ + "tool_id", + "status", + "provider", + "primary_observation_id", + "pass_eligible" + ], + "title": "ToolIdentityEvidence", + "type": "object" + }, + "ToolSemanticEvidence": { + "additionalProperties": false, + "description": "Public semantic assessment attached to action and capability facts.", + "properties": { + "authority": { + "$ref": "#/$defs/AuthoritySemanticEvidence" + }, + "binding": { + "$ref": "#/$defs/BindingSemanticEvidence" + }, + "conservative_effect": { + "enum": [ + "read", + "write", + "destructive", + "external_communication", + "financial_write", + "production_operation", + "privileged_data_access", + "code_execution", + "identity_access" + ], + "title": "Conservative Effect", + "type": "string" + }, + "effect": { + "$ref": "#/$defs/EffectSemanticEvidence" + }, + "identity": { + "anyOf": [ + { + "$ref": "#/$defs/ToolIdentityEvidence" + }, + { + "type": "null" + } + ], + "default": null + }, + "pass_eligible": { + "title": "Pass Eligible", + "type": "boolean" + } + }, + "required": [ + "conservative_effect", + "effect", + "authority", + "pass_eligible" + ], + "title": "ToolSemanticEvidence", + "type": "object" + } + }, + "$id": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/capability-lock-schema.v0.5.json", + "$ref": "#/$defs/CapabilityLockFileV1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Stable JSON Schema for static capability lock artifacts. Generated from agents_shipgate.schemas.capabilities. It is non-gating and is not part of report.json; release_decision.decision remains the only gate.", + "title": "Agents Shipgate Capability Lock v0.5" +} diff --git a/docs/capability-standard.md b/docs/capability-standard.md index 141ca9aa..f5fcc8f2 100644 --- a/docs/capability-standard.md +++ b/docs/capability-standard.md @@ -11,9 +11,9 @@ they do not create a second verdict. ## Current Versions -- Capability standard version: `0.3` -- Capability lock schema: [`capability-lock-schema.v0.4.json`](capability-lock-schema.v0.4.json) -- Capability lock diff schema: [`capability-lock-diff-schema.v0.5.json`](capability-lock-diff-schema.v0.5.json) +- Capability standard version: `0.4` +- Capability lock schema: [`capability-lock-schema.v0.5.json`](capability-lock-schema.v0.5.json) +- Capability lock diff schema: [`capability-lock-diff-schema.v0.6.json`](capability-lock-diff-schema.v0.6.json) - Frozen lock reference: [`capability-lock-schema.v0.2.json`](capability-lock-schema.v0.2.json) - Frozen lock-diff reference: [`capability-lock-diff-schema.v0.3.json`](capability-lock-diff-schema.v0.3.json) - Frozen experimental lock reference: [`capability-lock-schema.v0.1.json`](capability-lock-schema.v0.1.json) diff --git a/docs/codex-boundary-result-schema.v1.json b/docs/codex-boundary-result-schema.v1.json index a81a574a..7a39c6ee 100644 --- a/docs/codex-boundary-result-schema.v1.json +++ b/docs/codex-boundary-result-schema.v1.json @@ -369,7 +369,7 @@ "type": "string" }, "version": { - "default": "0.16.0b1", + "default": "0.16.0b2", "title": "Version", "type": "string" } diff --git a/docs/distribution.md b/docs/distribution.md index cdcaddfc..2ffaad89 100644 --- a/docs/distribution.md +++ b/docs/distribution.md @@ -38,7 +38,7 @@ exact wheel and signs `safety-qualification.json`: | Environment variable | Required value | |---|---| | `SAFETY_QUALIFICATION_WHEEL_URL` | HTTPS URL for the exact qualified wheel | -| `SAFETY_QUALIFICATION_WHEEL_FILENAME` | Safe wheel basename, for example `agents_shipgate-0.16.0b1-py3-none-any.whl` | +| `SAFETY_QUALIFICATION_WHEEL_FILENAME` | Safe wheel basename, for example `agents_shipgate-0.16.0b2-py3-none-any.whl` | | `SAFETY_QUALIFICATION_JSON_URL` | HTTPS URL for the production-qualified JSON artifact | | `SAFETY_QUALIFICATION_SIGSTORE_BUNDLE_URL` | HTTPS URL for that JSON artifact's Sigstore bundle | | `SAFETY_QUALIFICATION_SIGNER_IDENTITY` | Exact trusted certificate identity configured for qualification promotion | diff --git a/docs/examples.md b/docs/examples.md index f212c69e..e1e04515 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -69,8 +69,9 @@ The static scan fixtures write: - `agents-shipgate-reports/report.sarif` when requested or when using the GitHub Action The JSON output is the stable contract for tools and coding agents. See -[report-schema.v0.30.json](report-schema.v0.30.json) (current; emitted reports -carry `report_schema_version: "0.30"`, adding normalized semantic assessments +[report-schema.v0.31.json](report-schema.v0.31.json) (current; emitted reports +carry `report_schema_version: "0.31"`, adding a root-reachable binding graph +and retaining normalized semantic assessments and evidence-backed pass coverage while preserving policy-pack routing metadata in `findings[].policy_routing`; v0.29 is frozen at [report-schema.v0.29.json](report-schema.v0.29.json)). diff --git a/docs/examples/capability-lock-diff.v0.6.example.json b/docs/examples/capability-lock-diff.v0.6.example.json new file mode 100644 index 00000000..a9cc055a --- /dev/null +++ b/docs/examples/capability-lock-diff.v0.6.example.json @@ -0,0 +1,155 @@ +{ + "capability_lock_diff_schema_version": "0.6", + "experimental": false, + "base": { + "path": ".agents-shipgate/capabilities.lock.json", + "capability_lock_schema_version": "0.5", + "semantic_capability_set_hash": "2222222222222222222222222222222222222222222222222222222222222222", + "evidence_set_hash": "3333333333333333333333333333333333333333333333333333333333333333", + "source_set_hash": "4444444444444444444444444444444444444444444444444444444444444444", + "capability_count": 1 + }, + "head": { + "path": "agents-shipgate-reports/capabilities.lock.json", + "capability_lock_schema_version": "0.5", + "semantic_capability_set_hash": "5555555555555555555555555555555555555555555555555555555555555555", + "evidence_set_hash": "6666666666666666666666666666666666666666666666666666666666666666", + "source_set_hash": "4444444444444444444444444444444444444444444444444444444444444444", + "capability_count": 2 + }, + "summary": { + "added": 1, + "removed": 0, + "reidentified": 0, + "changed": 0, + "evidence_changed": 0, + "unchanged": 1 + }, + "added": [ + { + "id": "cap_999999999999", + "identity": { + "agent_id": "agent:support", + "tool_id": "tool:refunds.create", + "tool_name": "refunds.create", + "provider": "openapi", + "operation": "financial_write", + "subject_kind": "action", + "resource": [ + "refund" + ], + "scope": [ + "refunds:write" + ] + }, + "effect": { + "effect": "financial_write", + "externally_visible": true, + "handles_sensitive_data": true, + "financial": true, + "code_execution": false, + "reversibility": "irreversible", + "idempotency_known": false, + "high_risk": true + }, + "authority": { + "auth_type": "oauth2", + "credential_mode": "delegated", + "source": "manifest", + "scopes": [ + "refunds:write" + ], + "broad_scopes": [] + }, + "controls": { + "approval_required": true, + "approval_threshold": "security", + "confirmation_required": true, + "safeguard_idempotency": true, + "safeguard_audit_log": true, + "safeguard_rollback": false, + "safeguard_dry_run": null, + "evidence_owner": "payments-platform", + "evidence_runbook": "runbooks/refunds.md", + "evidence_approval_ticket": null + }, + "evidence": { + "source_type": "openapi", + "source_id": "support_api", + "source_ref": "tools/support.openapi.yaml", + "source_location": null, + "source_path": "tools/support.openapi.yaml", + "source_start_line": 88, + "source_end_line": null, + "source_start_column": null, + "source_pointer": "/paths/~1refunds/post", + "provenance_kind": "static_declaration", + "confidence": "high" + }, + "semantic_assessment": { + "conservative_effect": "financial_write", + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "financial_write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "manifest.action_surface", + "source_pointer": "/action_surface/actions/0/effect", + "evidence": { + "effect": "financial_write" + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "scoped", + "auth_type": "oauth2", + "credential_mode": "delegated", + "scopes": [ + "refunds:write" + ], + "claims": [ + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "manifest.action_surface", + "source_pointer": "/action_surface/actions/0/authority", + "evidence": { + "mode": "scoped" + } + } + ], + "issues": [] + }, + "pass_eligible": true + }, + "risk_tags": [ + "financial_action", + "high_impact", + "write" + ], + "hashes": { + "identity_hash": "9999999999999999999999999999999999999999999999999999999999999999", + "binding_hash": "abababababababababababababababababababababababababababababababab", + "effect_hash": "8888888888888888888888888888888888888888888888888888888888888888", + "authority_hash": "7777777777777777777777777777777777777777777777777777777777777777", + "control_hash": "6666666666666666666666666666666666666666666666666666666666666666", + "schema_hash": "5555555555555555555555555555555555555555555555555555555555555555", + "risk_hash": "4444444444444444444444444444444444444444444444444444444444444444", + "evidence_hash": "3333333333333333333333333333333333333333333333333333333333333333" + } + } + ], + "removed": [], + "reidentified": [], + "changed": [], + "evidence_changed": [] +} diff --git a/docs/examples/capability-lock.v0.5.example.json b/docs/examples/capability-lock.v0.5.example.json new file mode 100644 index 00000000..86e53662 --- /dev/null +++ b/docs/examples/capability-lock.v0.5.example.json @@ -0,0 +1,153 @@ +{ + "capability_lock_schema_version": "0.5", + "experimental": false, + "cli_version": "0.16.0b2", + "source": { + "config_path": "shipgate.yaml", + "manifest_dir": ".", + "project_name": "support-agent", + "agent_id": "agent:support", + "agent_name": "Support Agent", + "environment_target": "production", + "tool_count": 1, + "toolkit_bound_count": 0, + "source_count": 1, + "source_warning_count": 0, + "plugins_enabled": true + }, + "summary": { + "capability_count": 1, + "high_risk_count": 0, + "broad_scope_count": 0, + "write_count": 0, + "external_communication_count": 0, + "financial_count": 0, + "code_execution_count": 0 + }, + "hashes": { + "semantic_capability_set_hash": "2222222222222222222222222222222222222222222222222222222222222222", + "evidence_set_hash": "3333333333333333333333333333333333333333333333333333333333333333", + "source_set_hash": "4444444444444444444444444444444444444444444444444444444444444444" + }, + "capabilities": [ + { + "id": "cap_aaaaaaaaaaaa", + "identity": { + "agent_id": "agent:support", + "tool_id": "tool:cases.search", + "tool_name": "cases.search", + "provider": "openapi", + "operation": "read", + "subject_kind": "action", + "resource": [ + "case" + ], + "scope": [ + "cases:read" + ] + }, + "effect": { + "effect": "read", + "externally_visible": false, + "handles_sensitive_data": false, + "financial": false, + "code_execution": false, + "reversibility": "reversible", + "idempotency_known": true, + "high_risk": false + }, + "authority": { + "auth_type": "oauth2", + "credential_mode": "delegated", + "source": "openapi", + "scopes": [ + "cases:read" + ], + "broad_scopes": [] + }, + "controls": { + "approval_required": false, + "approval_threshold": null, + "confirmation_required": false, + "safeguard_idempotency": true, + "safeguard_audit_log": true, + "safeguard_rollback": null, + "safeguard_dry_run": null, + "evidence_owner": "support-platform", + "evidence_runbook": "runbooks/cases.md", + "evidence_approval_ticket": null + }, + "evidence": { + "source_type": "openapi", + "source_id": "support_api", + "source_ref": "tools/support.openapi.yaml", + "source_location": null, + "source_path": "tools/support.openapi.yaml", + "source_start_line": 42, + "source_end_line": null, + "source_start_column": null, + "source_pointer": "/paths/~1cases/get", + "provenance_kind": "static_declaration", + "confidence": "high" + }, + "semantic_assessment": { + "conservative_effect": "read", + "effect": { + "status": "structural", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi.method", + "source_pointer": "/paths/~1cases/get", + "evidence": { + "method": "GET" + } + } + ], + "issues": [] + }, + "authority": { + "status": "structural", + "mode": "scoped", + "auth_type": "oauth2", + "credential_mode": "delegated", + "scopes": [ + "cases:read" + ], + "claims": [ + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi.security", + "source_pointer": "/paths/~1cases/get/security", + "evidence": { + "scopes": [ + "cases:read" + ] + } + } + ], + "issues": [] + }, + "pass_eligible": true + }, + "risk_tags": [], + "hashes": { + "identity_hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "binding_hash": "abababababababababababababababababababababababababababababababab", + "effect_hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "authority_hash": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "control_hash": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "schema_hash": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "risk_hash": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "evidence_hash": "1111111111111111111111111111111111111111111111111111111111111111" + } + } + ] +} diff --git a/docs/faq.md b/docs/faq.md index 048138c5..66d7aa18 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -102,7 +102,7 @@ schema. - **Markdown** — `agents-shipgate-reports/report.md`, for human review. - **JSON** — `agents-shipgate-reports/report.json`, machine-readable - (schema v0.30, current). Always parse this for programmatic use. + (schema v0.31, current). Always parse this for programmatic use. For release gating, read `release_decision.decision`; the legacy `summary.status` field is baseline-blind (kept for v0.7 callers). A `passed` decision requires complete, conflict-free static surface, effect, @@ -118,14 +118,14 @@ schema. ## What is the Release Evidence Packet? A reviewer-shaped synthesis of the scan, emitted alongside the report by -default. The packet is governed by [`docs/packet-schema.v0.9.json`](packet-schema.v0.9.json) +default. The packet is governed by [`docs/packet-schema.v0.10.json`](packet-schema.v0.10.json) and has fixed reviewer sections (release decision, evidence matrix, capability/intent, high-risk surface, tool-surface diff, action-surface diff, approval coverage, idempotency risk, scope coverage, memory isolation, human-in-the-loop, dynamic scenarios, and a `not_proven` section that always lists prompt robustness, runtime behavior, model correctness, and adversarial resistance verbatim). See -[STABILITY.md §Release Evidence Packet](../STABILITY.md#release-evidence-packet-v09). +[STABILITY.md §Release Evidence Packet](../STABILITY.md#release-evidence-packet-v010). Packet schema `0.9` adds semantic coverage and gap remediation, and packet §1 mirrors the report's `static_analysis_only: true`, `runtime_behavior_verified: false`, and canonical @@ -146,7 +146,7 @@ Skip emission with `--no-packet`; re-render later with ## Is it production-ready? v0.15.0 is the latest published pre-1.0 beta. The in-tree runtime is -`0.16.0b1`, which introduces the evidence-backed `passed` contract for beta +`0.16.0b2`, which adds root-reachable binding proof to the evidence-backed `passed` contract for beta qualification. The manifest schema remains stable across the 0.x series; see [`STABILITY.md`](../STABILITY.md). Public preview. diff --git a/docs/governance-benchmark-catalog-schema.v0.2.json b/docs/governance-benchmark-catalog-schema.v0.2.json index 6ca3e2fe..d9325883 100644 --- a/docs/governance-benchmark-catalog-schema.v0.2.json +++ b/docs/governance-benchmark-catalog-schema.v0.2.json @@ -31,6 +31,7 @@ "items": { "enum": [ "identity_hash", + "binding_hash", "effect_hash", "authority_hash", "control_hash", diff --git a/docs/manifest-v0.1.json b/docs/manifest-v0.1.json index 2024016c..21ba0304 100644 --- a/docs/manifest-v0.1.json +++ b/docs/manifest-v0.1.json @@ -552,6 +552,98 @@ "title": "ActionSurfaceConfig", "type": "object" }, + "AgentBindingDeclarationConfig": { + "additionalProperties": false, + "description": "Reviewed, closed-world tool and handoff set for one agent.", + "properties": { + "agent": { + "title": "Agent", + "type": "string" + }, + "complete": { + "const": true, + "default": true, + "title": "Complete", + "type": "boolean" + }, + "handoffs": { + "items": { + "type": "string" + }, + "title": "Handoffs", + "type": "array" + }, + "reason": { + "title": "Reason", + "type": "string" + }, + "tools": { + "items": { + "$ref": "#/$defs/BoundToolSelectorConfig" + }, + "title": "Tools", + "type": "array" + } + }, + "required": [ + "agent", + "reason" + ], + "title": "AgentBindingDeclarationConfig", + "type": "object" + }, + "AgentBindingRootConfig": { + "additionalProperties": false, + "description": "Exact selector for the statically reviewed root agent.", + "properties": { + "object": { + "title": "Object", + "type": "string" + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + } + }, + "required": [ + "object" + ], + "title": "AgentBindingRootConfig", + "type": "object" + }, + "AgentBindingsConfig": { + "additionalProperties": false, + "properties": { + "declarations": { + "items": { + "$ref": "#/$defs/AgentBindingDeclarationConfig" + }, + "title": "Declarations", + "type": "array" + }, + "root": { + "anyOf": [ + { + "$ref": "#/$defs/AgentBindingRootConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "AgentBindingsConfig", + "type": "object" + }, "AgentConfig": { "additionalProperties": false, "properties": { @@ -745,6 +837,69 @@ "title": "BaselineConfig", "type": "object" }, + "BoundToolSelectorConfig": { + "additionalProperties": false, + "description": "One canonical tool selector inside a reviewed binding declaration.", + "properties": { + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Type" + }, + "tool": { + "title": "Tool", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "tool" + ], + "title": "BoundToolSelectorConfig", + "type": "object" + }, "ChecksConfig": { "additionalProperties": false, "properties": { @@ -2215,6 +2370,9 @@ "agent": { "$ref": "#/$defs/AgentConfig" }, + "agent_bindings": { + "$ref": "#/$defs/AgentBindingsConfig" + }, "anthropic": { "anyOf": [ { diff --git a/docs/manifest-v0.1.md b/docs/manifest-v0.1.md index 50849957..1d54c5e5 100644 --- a/docs/manifest-v0.1.md +++ b/docs/manifest-v0.1.md @@ -398,6 +398,32 @@ belong to at most one binding. Equal names or providers never imply equivalence. Invalid, overlapping, or structurally conflicting bindings join nothing and prevent `passed`. +## Agent Binding Surface + +`agent_bindings` declares the exact root agent and, when framework wiring is +not completely statically visible, a reviewed closed-world tool/handoff set: + +```yaml +agent_bindings: + root: + source_id: ops_sdk + object: ops_assistant + declarations: + - agent: root + complete: true + tools: + - tool: docs.lookup + source_id: docs_tools + handoffs: [] + reason: reviewed against the deployed agent wiring +``` + +Catalog membership, action declarations, permissions, and controls never imply +binding. `complete` must be `true`; the declaration is exact and may not erase +positive structural edges. Empty `tools` and `handoffs` prove a zero-capability +root. These are human-reviewed claims and must never be inferred or auto-filled +by a coding agent. + ## Action Surface Diff The optional top-level `action_surface:` block adds reviewer-facing action diff --git a/docs/overview.md b/docs/overview.md index d8b4b491..59872af0 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -44,7 +44,7 @@ surface before production-like permissions are granted. - [Concepts](concepts.md) - [Manifest v0.1](manifest-v0.1.md) - [Check catalog](checks.md) -- [Report schema v0.30](report-schema.v0.30.json) (current; v0.29 frozen at [report-schema.v0.29.json](report-schema.v0.29.json), v0.28 frozen at [report-schema.v0.28.json](report-schema.v0.28.json), v0.27 frozen at [report-schema.v0.27.json](report-schema.v0.27.json)) +- [Report schema v0.31](report-schema.v0.31.json) (current; v0.30 frozen at [report-schema.v0.30.json](report-schema.v0.30.json), v0.29 frozen at [report-schema.v0.29.json](report-schema.v0.29.json)) - [Evidence-backed `passed` contract](passed-verdict-contract.md) - [Trust model](trust-model.md) - [Agent instructions](../AGENTS.md) diff --git a/docs/packet-schema.v0.10.json b/docs/packet-schema.v0.10.json new file mode 100644 index 00000000..e1d1a83e --- /dev/null +++ b/docs/packet-schema.v0.10.json @@ -0,0 +1,1988 @@ +{ + "$defs": { + "ActionSurfaceDiffSection": { + "additionalProperties": false, + "description": "\u00a73B \u2014 compact action-surface diff for reviewers.", + "properties": { + "base_kind": { + "default": "none", + "title": "Base Kind", + "type": "string" + }, + "blocking_reasons": { + "items": { + "type": "string" + }, + "title": "Blocking Reasons", + "type": "array" + }, + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "highlights": { + "items": { + "type": "string" + }, + "title": "Highlights", + "type": "array" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + }, + "summary": { + "$ref": "#/$defs/ActionSurfaceDiffSummary" + } + }, + "required": [ + "status" + ], + "title": "ActionSurfaceDiffSection", + "type": "object" + }, + "ActionSurfaceDiffSummary": { + "additionalProperties": false, + "properties": { + "actions_added": { + "default": 0, + "title": "Actions Added", + "type": "integer" + }, + "actions_modified": { + "default": 0, + "title": "Actions Modified", + "type": "integer" + }, + "actions_removed": { + "default": 0, + "title": "Actions Removed", + "type": "integer" + }, + "approvals_removed": { + "default": 0, + "title": "Approvals Removed", + "type": "integer" + }, + "blocking_findings": { + "default": 0, + "title": "Blocking Findings", + "type": "integer" + }, + "effect_escalations": { + "default": 0, + "title": "Effect Escalations", + "type": "integer" + }, + "input_schema_expansions": { + "default": 0, + "title": "Input Schema Expansions", + "type": "integer" + }, + "risk_tags_added": { + "default": 0, + "title": "Risk Tags Added", + "type": "integer" + }, + "safeguards_removed": { + "default": 0, + "title": "Safeguards Removed", + "type": "integer" + }, + "scope_expansions": { + "default": 0, + "title": "Scope Expansions", + "type": "integer" + } + }, + "title": "ActionSurfaceDiffSummary", + "type": "object" + }, + "ApprovalCoverageRow": { + "additionalProperties": false, + "properties": { + "declared": { + "title": "Declared", + "type": "boolean" + }, + "gap_finding_ids": { + "items": { + "type": "string" + }, + "title": "Gap Finding Ids", + "type": "array" + }, + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + }, + "tool": { + "title": "Tool", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "tool", + "declared" + ], + "title": "ApprovalCoverageRow", + "type": "object" + }, + "ApprovalCoverageSection": { + "additionalProperties": false, + "description": "\u00a74 \u2014 approval policy coverage.", + "properties": { + "gap_findings": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Gap Findings", + "type": "array" + }, + "rows": { + "items": { + "$ref": "#/$defs/ApprovalCoverageRow" + }, + "title": "Rows", + "type": "array" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "ApprovalCoverageSection", + "type": "object" + }, + "BaselineDelta": { + "properties": { + "enabled": { + "title": "Enabled", + "type": "boolean" + }, + "matched_count": { + "default": 0, + "title": "Matched Count", + "type": "integer" + }, + "new_count": { + "default": 0, + "title": "New Count", + "type": "integer" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "resolved_count": { + "default": 0, + "title": "Resolved Count", + "type": "integer" + } + }, + "required": [ + "enabled" + ], + "title": "BaselineDelta", + "type": "object" + }, + "BindingCoverageDecision": { + "properties": { + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible": { + "default": false, + "title": "Pass Eligible", + "type": "boolean" + }, + "possible_tools": { + "default": 0, + "title": "Possible Tools", + "type": "integer" + }, + "reachable_tools": { + "default": 0, + "title": "Reachable Tools", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "total_catalog_tools": { + "default": 0, + "title": "Total Catalog Tools", + "type": "integer" + }, + "unbound_tools": { + "default": 0, + "title": "Unbound Tools", + "type": "integer" + } + }, + "title": "BindingCoverageDecision", + "type": "object" + }, + "CapabilityIntentDiff": { + "additionalProperties": false, + "description": "\u00a72 \u2014 capability \u2194 intent diff.", + "properties": { + "declared_purpose": { + "items": { + "type": "string" + }, + "title": "Declared Purpose", + "type": "array" + }, + "divergence_findings": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Divergence Findings", + "type": "array" + }, + "observed_tool_ids": { + "items": { + "type": "string" + }, + "title": "Observed Tool Ids", + "type": "array" + }, + "observed_tools": { + "items": { + "type": "string" + }, + "title": "Observed Tools", + "type": "array" + }, + "prohibited_actions": { + "items": { + "type": "string" + }, + "title": "Prohibited Actions", + "type": "array" + }, + "rows": { + "items": { + "$ref": "#/$defs/CapabilityIntentRow" + }, + "title": "Rows", + "type": "array" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "CapabilityIntentDiff", + "type": "object" + }, + "CapabilityIntentRow": { + "additionalProperties": false, + "description": "One declared/observed pair in the \u00a72 capability \u2194 intent diff.", + "properties": { + "declared": { + "items": { + "type": "string" + }, + "title": "Declared", + "type": "array" + }, + "divergent": { + "items": { + "type": "string" + }, + "title": "Divergent", + "type": "array" + }, + "label": { + "title": "Label", + "type": "string" + }, + "observed": { + "items": { + "type": "string" + }, + "title": "Observed", + "type": "array" + } + }, + "required": [ + "label" + ], + "title": "CapabilityIntentRow", + "type": "object" + }, + "CapabilityTraceEvidenceSummary": { + "additionalProperties": false, + "description": "Deterministic counts for opt-in local runtime trace evidence.", + "properties": { + "agent_trace_count": { + "default": 0, + "title": "Agent Trace Count", + "type": "integer" + }, + "api_trace_count": { + "default": 0, + "title": "Api Trace Count", + "type": "integer" + }, + "approval_trace_count": { + "default": 0, + "title": "Approval Trace Count", + "type": "integer" + }, + "matched_trace_count": { + "default": 0, + "title": "Matched Trace Count", + "type": "integer" + }, + "source_count": { + "default": 0, + "title": "Source Count", + "type": "integer" + }, + "trace_count": { + "default": 0, + "title": "Trace Count", + "type": "integer" + }, + "unmatched_trace_count": { + "default": 0, + "title": "Unmatched Trace Count", + "type": "integer" + }, + "warning_count": { + "default": 0, + "title": "Warning Count", + "type": "integer" + } + }, + "title": "CapabilityTraceEvidenceSummary", + "type": "object" + }, + "DynamicScenarioRequirement": { + "additionalProperties": false, + "properties": { + "finding_ids": { + "items": { + "type": "string" + }, + "title": "Finding Ids", + "type": "array" + }, + "scenario": { + "title": "Scenario", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "scenario", + "why" + ], + "title": "DynamicScenarioRequirement", + "type": "object" + }, + "DynamicScenariosSection": { + "additionalProperties": false, + "description": "\u00a79 \u2014 required dynamic scenarios.", + "properties": { + "scenarios": { + "items": { + "$ref": "#/$defs/DynamicScenarioRequirement" + }, + "title": "Scenarios", + "type": "array" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "DynamicScenariosSection", + "type": "object" + }, + "EvidenceCoverageDecision": { + "properties": { + "binding_coverage": { + "$ref": "#/$defs/BindingCoverageDecision" + }, + "evidence_gaps": { + "items": { + "$ref": "#/$defs/EvidenceGap" + }, + "title": "Evidence Gaps", + "type": "array" + }, + "human_review_recommended": { + "title": "Human Review Recommended", + "type": "boolean" + }, + "identity_coverage": { + "$ref": "#/$defs/IdentityCoverageDecision" + }, + "level": { + "title": "Level", + "type": "string" + }, + "low_confidence_tool_count": { + "title": "Low Confidence Tool Count", + "type": "integer" + }, + "semantic_coverage": { + "$ref": "#/$defs/SemanticCoverageDecision" + }, + "source_warning_count": { + "title": "Source Warning Count", + "type": "integer" + } + }, + "required": [ + "level", + "human_review_recommended", + "source_warning_count", + "low_confidence_tool_count" + ], + "title": "EvidenceCoverageDecision", + "type": "object" + }, + "EvidenceGap": { + "description": "v0.26: one structured row per measurable evidence gap.\n\n``insufficient_evidence`` previously diagnosed without prescribing;\neach gap names the degraded subject and the specific next action\nthat raises extraction confidence. Purely explanatory \u2014 gating\nstill uses only the counts (the gap list is a projection of them).", + "properties": { + "kind": { + "enum": [ + "low_confidence_tool", + "source_warning", + "incomplete_surface", + "missing_effect_evidence", + "inferred_effect_only", + "conflicting_effect_evidence", + "missing_authority_evidence", + "partial_authority_evidence", + "conflicting_authority_evidence", + "invalid_semantic_annotation", + "incomplete_tool_identity", + "conflicting_tool_identity", + "unresolved_tool_selector", + "ambiguous_tool_selector", + "ambiguous_legacy_tool_identity", + "invalid_tool_binding", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation" + ], + "title": "Kind", + "type": "string" + }, + "next_action": { + "$ref": "#/$defs/EvidenceGapAction" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Type" + }, + "subject": { + "title": "Subject", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "kind", + "subject", + "why", + "next_action" + ], + "title": "EvidenceGap", + "type": "object" + }, + "EvidenceGapAction": { + "description": "One concrete, mechanically-executable step that closes a gap.\n\nMirrors the agent-mode ``next_actions[]`` error shape\n(``kind``/``command``/``path``/``why``/``expects``) so agents reuse\none routing vocabulary across error recovery and evidence repair.", + "properties": { + "accepted_values": { + "items": { + "type": "string" + }, + "title": "Accepted Values", + "type": "array" + }, + "auto_apply": { + "const": false, + "default": false, + "title": "Auto Apply", + "type": "boolean" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "declaration_template": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Declaration Template" + }, + "expects": { + "title": "Expects", + "type": "string" + }, + "kind": { + "enum": [ + "declare_tool_inventory", + "provide_source", + "review_warning", + "declare_action_effect", + "declare_action_authority", + "provide_complete_inventory", + "resolve_semantic_conflict", + "declare_source_identity", + "qualify_tool_selector", + "provide_tool_binding", + "resolve_tool_identity_conflict", + "regenerate_identity_artifact", + "declare_agent_root", + "declare_agent_bindings", + "provide_static_binding_source", + "provide_complete_binding_graph", + "resolve_binding_conflict", + "regenerate_binding_artifact" + ], + "title": "Kind", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "requires_human_review": { + "const": true, + "default": true, + "title": "Requires Human Review", + "type": "boolean" + }, + "suggested_patch_kind": { + "const": "manual", + "default": "manual", + "title": "Suggested Patch Kind", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "kind", + "why", + "expects" + ], + "title": "EvidenceGapAction", + "type": "object" + }, + "EvidenceMatrixRow": { + "additionalProperties": false, + "description": "One compact row in the packet-only evidence matrix.\n\nThe row is a derived review aid only. ``blocking_findings`` and\n``review_items`` are subsets copied from ``release_decision``; the\nmatrix never reclassifies findings or contributes to gate behavior.", + "properties": { + "blocking_findings": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Blocking Findings", + "type": "array" + }, + "confidence": { + "default": "unknown", + "enum": [ + "high", + "medium", + "low", + "mixed", + "unknown" + ], + "title": "Confidence", + "type": "string" + }, + "domain": { + "enum": [ + "Inventory", + "Schema", + "Auth", + "Approval", + "Confirmation", + "Idempotency", + "Side effects", + "Memory isolation", + "Human-in-the-loop evidence", + "Prompt/scope alignment", + "Retry/timeout", + "Baseline debt", + "Action-surface policy" + ], + "title": "Domain", + "type": "string" + }, + "evidence_present": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Evidence Present", + "type": "string" + }, + "evidence_source": { + "items": { + "type": "string" + }, + "title": "Evidence Source", + "type": "array" + }, + "missing_controls": { + "items": { + "type": "string" + }, + "title": "Missing Controls", + "type": "array" + }, + "review_items": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Review Items", + "type": "array" + } + }, + "required": [ + "domain", + "evidence_present" + ], + "title": "EvidenceMatrixRow", + "type": "object" + }, + "EvidenceMatrixSection": { + "additionalProperties": false, + "description": "Compact packet-only review matrix derived from public report JSON.", + "properties": { + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "rows": { + "items": { + "$ref": "#/$defs/EvidenceMatrixRow" + }, + "title": "Rows", + "type": "array" + } + }, + "title": "EvidenceMatrixSection", + "type": "object" + }, + "FailPolicy": { + "properties": { + "ci_mode": { + "title": "Ci Mode", + "type": "string" + }, + "exit_code": { + "title": "Exit Code", + "type": "integer" + }, + "fail_on": { + "items": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "type": "string" + }, + "title": "Fail On", + "type": "array" + }, + "new_findings_only": { + "default": false, + "title": "New Findings Only", + "type": "boolean" + }, + "would_fail_ci": { + "title": "Would Fail Ci", + "type": "boolean" + } + }, + "required": [ + "ci_mode", + "would_fail_ci", + "exit_code" + ], + "title": "FailPolicy", + "type": "object" + }, + "HighRiskSurfaceSection": { + "additionalProperties": false, + "description": "\u00a73 \u2014 high-risk tool surface.", + "properties": { + "high_risk_count": { + "default": 0, + "title": "High Risk Count", + "type": "integer" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + }, + "tools": { + "items": { + "$ref": "#/$defs/HighRiskToolEntry" + }, + "title": "Tools", + "type": "array" + }, + "total_tools": { + "default": 0, + "title": "Total Tools", + "type": "integer" + } + }, + "required": [ + "status" + ], + "title": "HighRiskSurfaceSection", + "type": "object" + }, + "HighRiskToolEntry": { + "additionalProperties": false, + "properties": { + "has_approval_policy": { + "default": false, + "title": "Has Approval Policy", + "type": "boolean" + }, + "has_idempotency_policy": { + "default": false, + "title": "Has Idempotency Policy", + "type": "boolean" + }, + "name": { + "title": "Name", + "type": "string" + }, + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "risk_tags": { + "items": { + "type": "string" + }, + "title": "Risk Tags", + "type": "array" + }, + "source_type": { + "title": "Source Type", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "name", + "source_type" + ], + "title": "HighRiskToolEntry", + "type": "object" + }, + "HitlSourceProvenance": { + "additionalProperties": false, + "properties": { + "detail": { + "title": "Detail", + "type": "string" + }, + "location": { + "title": "Location", + "type": "string" + }, + "ref": { + "title": "Ref", + "type": "string" + }, + "status": { + "enum": [ + "requirement_only", + "expected_but_absent", + "source_load_failed", + "loaded", + "loaded_with_warnings" + ], + "title": "Status", + "type": "string" + }, + "type": { + "enum": [ + "approval_trace", + "agent_trace", + "override_log", + "high_risk_exclusion", + "promotion_criteria", + "manifest_requirement" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "ref", + "location", + "status", + "detail" + ], + "title": "HitlSourceProvenance", + "type": "object" + }, + "HumanInTheLoopEvidence": { + "additionalProperties": false, + "description": "\u00a78 \u2014 human-in-the-loop evidence.\n\n``trace_findings`` is kept for packet v0.1 compatibility. It now\ncarries HITL evidence gaps surfaced as findings, including the\noriginal trace-derived gaps and validation-config evidence gaps.", + "properties": { + "approval_required_tools": { + "items": { + "type": "string" + }, + "title": "Approval Required Tools", + "type": "array" + }, + "capability_trace_refs": { + "items": { + "type": "string" + }, + "title": "Capability Trace Refs", + "type": "array" + }, + "capability_trace_summary": { + "$ref": "#/$defs/CapabilityTraceEvidenceSummary" + }, + "confirmation_required_tools": { + "items": { + "type": "string" + }, + "title": "Confirmation Required Tools", + "type": "array" + }, + "human_review_recommended": { + "default": false, + "title": "Human Review Recommended", + "type": "boolean" + }, + "is_configured": { + "default": false, + "title": "Is Configured", + "type": "boolean" + }, + "provenance_mode": { + "default": "fresh_scan", + "enum": [ + "fresh_scan", + "rebuilt_from_findings", + "unavailable" + ], + "title": "Provenance Mode", + "type": "string" + }, + "runtime_control_disclaimer": { + "default": "HITL evidence is local review evidence only. Missing local evidence does not prove a runtime control is absent, and present local evidence does not certify runtime enforcement.", + "title": "Runtime Control Disclaimer", + "type": "string" + }, + "source_provenance": { + "items": { + "$ref": "#/$defs/HitlSourceProvenance" + }, + "title": "Source Provenance", + "type": "array" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + }, + "trace_findings": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Trace Findings", + "type": "array" + } + }, + "required": [ + "status" + ], + "title": "HumanInTheLoopEvidence", + "type": "object" + }, + "IdempotencyRiskSection": { + "additionalProperties": false, + "description": "\u00a75 \u2014 idempotency / retry risk.", + "properties": { + "gap_findings": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Gap Findings", + "type": "array" + }, + "retry_policy_declared": { + "default": false, + "title": "Retry Policy Declared", + "type": "boolean" + }, + "rows": { + "items": { + "$ref": "#/$defs/IdempotencyRow" + }, + "title": "Rows", + "type": "array" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "IdempotencyRiskSection", + "type": "object" + }, + "IdempotencyRow": { + "additionalProperties": false, + "properties": { + "declared": { + "title": "Declared", + "type": "boolean" + }, + "gap_finding_ids": { + "items": { + "type": "string" + }, + "title": "Gap Finding Ids", + "type": "array" + }, + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + }, + "tool": { + "title": "Tool", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "tool", + "declared" + ], + "title": "IdempotencyRow", + "type": "object" + }, + "IdentityCoverageDecision": { + "properties": { + "ambiguous_name_count": { + "default": 0, + "title": "Ambiguous Name Count", + "type": "integer" + }, + "bound_tools": { + "default": 0, + "title": "Bound Tools", + "type": "integer" + }, + "canonical_tools": { + "default": 0, + "title": "Canonical Tools", + "type": "integer" + }, + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible_tools": { + "default": 0, + "title": "Pass Eligible Tools", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "total_observations": { + "default": 0, + "title": "Total Observations", + "type": "integer" + } + }, + "title": "IdentityCoverageDecision", + "type": "object" + }, + "MemoryIsolationStatus": { + "additionalProperties": false, + "description": "\u00a77 \u2014 memory isolation. v0.1 always renders ``is_declared=False``;\na future manifest schema may add ``agent.memory`` and populate this\nsection. Until then the structural slot is preserved so packets\nhave a consistent shape across versions.", + "properties": { + "is_declared": { + "default": false, + "title": "Is Declared", + "type": "boolean" + }, + "notes": { + "default": "Manifest does not declare a memory isolation policy. The current manifest schema (v0.1) has no agent.memory field. See \u00a710 for the residual review item.", + "title": "Notes", + "type": "string" + }, + "status": { + "default": "not_declared", + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + } + }, + "title": "MemoryIsolationStatus", + "type": "object" + }, + "NotProvenItem": { + "additionalProperties": false, + "properties": { + "body": { + "title": "Body", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label", + "body" + ], + "title": "NotProvenItem", + "type": "object" + }, + "NotProvenSection": { + "additionalProperties": false, + "description": "\u00a710 \u2014 what Shipgate did NOT prove. Combines unconditional\ndisclaimers with per-run residuals so reviewers can see both the\nstatic scope and what's missing for this specific scan.", + "properties": { + "additional_residuals": { + "items": { + "type": "string" + }, + "title": "Additional Residuals", + "type": "array" + }, + "headline": { + "title": "Headline", + "type": "string" + }, + "low_confidence_tools": { + "items": { + "type": "string" + }, + "title": "Low Confidence Tools", + "type": "array" + }, + "source_warnings": { + "items": { + "type": "string" + }, + "title": "Source Warnings", + "type": "array" + }, + "suppressed_finding_ids": { + "items": { + "type": "string" + }, + "title": "Suppressed Finding Ids", + "type": "array" + }, + "unconditional": { + "items": { + "$ref": "#/$defs/NotProvenItem" + }, + "title": "Unconditional", + "type": "array" + } + }, + "required": [ + "headline" + ], + "title": "NotProvenSection", + "type": "object" + }, + "ReleaseDecisionItem": { + "properties": { + "baseline_status": { + "anyOf": [ + { + "enum": [ + "new", + "matched", + "resolved" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Baseline Status" + }, + "blocks_release": { + "default": false, + "title": "Blocks Release", + "type": "boolean" + }, + "capability_trace_refs": { + "items": { + "type": "string" + }, + "title": "Capability Trace Refs", + "type": "array" + }, + "check_id": { + "title": "Check Id", + "type": "string" + }, + "fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fingerprint" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "policy_evidence_source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "severity": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Severity", + "type": "string" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "title": { + "title": "Title", + "type": "string" + } + }, + "required": [ + "check_id", + "severity", + "title" + ], + "title": "ReleaseDecisionItem", + "type": "object" + }, + "ReleaseDecisionSection": { + "additionalProperties": false, + "description": "\u00a71 \u2014 release decision. Verdict derives from\n``release_decision.decision`` only; ``fail_policy`` is rendered as\nseparate CI behavior metadata, never as the verdict source.", + "properties": { + "baseline_delta": { + "$ref": "#/$defs/BaselineDelta" + }, + "blockers": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Blockers", + "type": "array" + }, + "decision": { + "enum": [ + "blocked", + "review_required", + "insufficient_evidence", + "passed" + ], + "title": "Decision", + "type": "string" + }, + "evidence_coverage": { + "$ref": "#/$defs/EvidenceCoverageDecision" + }, + "fail_policy": { + "$ref": "#/$defs/FailPolicy" + }, + "reason": { + "title": "Reason", + "type": "string" + }, + "review_items": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Review Items", + "type": "array" + }, + "runtime_behavior_verified": { + "const": false, + "default": false, + "title": "Runtime Behavior Verified", + "type": "boolean" + }, + "static_analysis_only": { + "const": true, + "default": true, + "title": "Static Analysis Only", + "type": "boolean" + }, + "static_verdict_disclaimer": { + "default": "This verdict covers deterministic static evidence only. Agents Shipgate did not execute the agent or prove runtime behavior, tool routing, credential enforcement, or safety.", + "title": "Static Verdict Disclaimer", + "type": "string" + }, + "verdict": { + "enum": [ + "PASSED", + "REVIEW REQUIRED", + "INSUFFICIENT EVIDENCE", + "BLOCKED" + ], + "title": "Verdict", + "type": "string" + } + }, + "required": [ + "decision", + "verdict", + "reason", + "evidence_coverage", + "baseline_delta", + "fail_policy" + ], + "title": "ReleaseDecisionSection", + "type": "object" + }, + "ScopeCoverageRow": { + "additionalProperties": false, + "properties": { + "declared": { + "title": "Declared", + "type": "boolean" + }, + "scope": { + "title": "Scope", + "type": "string" + }, + "used_by_tool_ids": { + "items": { + "type": "string" + }, + "title": "Used By Tool Ids", + "type": "array" + }, + "used_by_tools": { + "items": { + "type": "string" + }, + "title": "Used By Tools", + "type": "array" + } + }, + "required": [ + "scope", + "declared" + ], + "title": "ScopeCoverageRow", + "type": "object" + }, + "ScopeCoverageSection": { + "additionalProperties": false, + "description": "\u00a76 \u2014 scope coverage.", + "properties": { + "declared_scopes": { + "items": { + "type": "string" + }, + "title": "Declared Scopes", + "type": "array" + }, + "gap_findings": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Gap Findings", + "type": "array" + }, + "missing_declared": { + "items": { + "type": "string" + }, + "title": "Missing Declared", + "type": "array" + }, + "rows": { + "items": { + "$ref": "#/$defs/ScopeCoverageRow" + }, + "title": "Rows", + "type": "array" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + }, + "unused_declared": { + "items": { + "type": "string" + }, + "title": "Unused Declared", + "type": "array" + } + }, + "required": [ + "status" + ], + "title": "ScopeCoverageSection", + "type": "object" + }, + "SemanticCoverageDecision": { + "description": "v0.29 pass eligibility across the normalized action surface.\n\nUnlike extraction-confidence thresholds, semantic gaps are\nzero-tolerance: any non-pass-eligible unknown/partial/conflicting\ndimension prevents ``passed``. Known authority review concerns (for\nexample ambient or unscoped credentials) are counted separately so\nthey deterministically route to ``review_required`` rather than\n``insufficient_evidence``.", + "properties": { + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible_actions": { + "default": 0, + "title": "Pass Eligible Actions", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "review_concern_count": { + "default": 0, + "title": "Review Concern Count", + "type": "integer" + }, + "total_actions": { + "default": 0, + "title": "Total Actions", + "type": "integer" + } + }, + "title": "SemanticCoverageDecision", + "type": "object" + }, + "SourceReference": { + "additionalProperties": true, + "properties": { + "end_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "End Line" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pointer" + }, + "ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ref" + }, + "start_column": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Start Column" + }, + "start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Start Line" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "SourceReference", + "type": "object" + }, + "ToolSurfaceDiffSection": { + "additionalProperties": false, + "description": "\u00a73A \u2014 compact tool-surface diff for reviewers.", + "properties": { + "base_kind": { + "default": "none", + "title": "Base Kind", + "type": "string" + }, + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "highlights": { + "items": { + "type": "string" + }, + "title": "Highlights", + "type": "array" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "status": { + "enum": [ + "covered", + "partial", + "not_declared", + "missing", + "informational" + ], + "title": "Status", + "type": "string" + }, + "summary": { + "$ref": "#/$defs/ToolSurfaceDiffSummary" + } + }, + "required": [ + "status" + ], + "title": "ToolSurfaceDiffSection", + "type": "object" + }, + "ToolSurfaceDiffSummary": { + "additionalProperties": false, + "properties": { + "accepted_debt": { + "default": 0, + "title": "Accepted Debt", + "type": "integer" + }, + "controls_added": { + "default": 0, + "title": "Controls Added", + "type": "integer" + }, + "controls_removed": { + "default": 0, + "title": "Controls Removed", + "type": "integer" + }, + "metadata_changes": { + "default": 0, + "title": "Metadata Changes", + "type": "integer" + }, + "new_findings": { + "default": 0, + "title": "New Findings", + "type": "integer" + }, + "new_high_risk_effects": { + "default": 0, + "title": "New High Risk Effects", + "type": "integer" + }, + "new_scopes": { + "default": 0, + "title": "New Scopes", + "type": "integer" + }, + "policy_drift_items": { + "default": 0, + "title": "Policy Drift Items", + "type": "integer" + }, + "removed_high_risk_effects": { + "default": 0, + "title": "Removed High Risk Effects", + "type": "integer" + }, + "removed_scopes": { + "default": 0, + "title": "Removed Scopes", + "type": "integer" + }, + "resolved_findings": { + "default": 0, + "title": "Resolved Findings", + "type": "integer" + }, + "tools_added": { + "default": 0, + "title": "Tools Added", + "type": "integer" + }, + "tools_changed": { + "default": 0, + "title": "Tools Changed", + "type": "integer" + }, + "tools_removed": { + "default": 0, + "title": "Tools Removed", + "type": "integer" + }, + "unchanged_findings": { + "default": 0, + "title": "Unchanged Findings", + "type": "integer" + } + }, + "title": "ToolSurfaceDiffSummary", + "type": "object" + } + }, + "$id": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/packet-schema.v0.10.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "JSON Schema for packet.json. Generated from agents_shipgate.schemas.packet.EvidencePacket. Do not edit by hand.", + "properties": { + "action_surface_diff": { + "$ref": "#/$defs/ActionSurfaceDiffSection" + }, + "agent": { + "additionalProperties": true, + "title": "Agent", + "type": "object" + }, + "approval_coverage": { + "$ref": "#/$defs/ApprovalCoverageSection" + }, + "capability_intent": { + "$ref": "#/$defs/CapabilityIntentDiff" + }, + "dynamic_scenarios": { + "$ref": "#/$defs/DynamicScenariosSection" + }, + "environment": { + "additionalProperties": true, + "title": "Environment", + "type": "object" + }, + "evidence_matrix": { + "$ref": "#/$defs/EvidenceMatrixSection" + }, + "generated_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Generated At" + }, + "high_risk_surface": { + "$ref": "#/$defs/HighRiskSurfaceSection" + }, + "human_in_the_loop": { + "$ref": "#/$defs/HumanInTheLoopEvidence" + }, + "idempotency_risk": { + "$ref": "#/$defs/IdempotencyRiskSection" + }, + "memory_isolation": { + "$ref": "#/$defs/MemoryIsolationStatus" + }, + "not_proven": { + "$ref": "#/$defs/NotProvenSection" + }, + "packet_schema_version": { + "const": "0.10", + "default": "0.10", + "title": "Packet Schema Version", + "type": "string" + }, + "project": { + "additionalProperties": true, + "title": "Project", + "type": "object" + }, + "release_decision": { + "$ref": "#/$defs/ReleaseDecisionSection" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "scope_coverage": { + "$ref": "#/$defs/ScopeCoverageSection" + }, + "tool_surface_diff": { + "$ref": "#/$defs/ToolSurfaceDiffSection" + } + }, + "required": [ + "run_id", + "release_decision", + "capability_intent", + "high_risk_surface", + "approval_coverage", + "idempotency_risk", + "scope_coverage", + "memory_isolation", + "human_in_the_loop", + "dynamic_scenarios", + "not_proven" + ], + "title": "Agents Shipgate Release Evidence Packet v0.10", + "type": "object" +} diff --git a/docs/passed-verdict-contract.md b/docs/passed-verdict-contract.md index 4da6527f..0a9772d2 100644 --- a/docs/passed-verdict-contract.md +++ b/docs/passed-verdict-contract.md @@ -1,9 +1,10 @@ # Evidence-backed `passed` verdict -Starting with the Agents Shipgate `0.16.0b1` runtime (contract v12, report -schema v0.30), `release_decision.decision: passed` means -that every in-scope capability has complete, conflict-free static surface, -effect, and authority evidence, all applicable controls were evaluated, and no +Starting with the Agents Shipgate `0.16.0b2` runtime (contract v13, report +schema v0.31), `release_decision.decision: passed` means the configured root +agent and its complete reachable tool/handoff graph were statically proven, +and every reachable capability has complete, conflict-free static identity, +binding, effect, and authority evidence, all applicable controls were evaluated, and no policy condition requires review. It does not prove runtime agent behavior or runtime enforcement. @@ -12,6 +13,8 @@ runtime enforcement. An action is pass-eligible only when all of the following hold: - its source surface is completely enumerable and extracted at high confidence; +- it is reachable from the unambiguously selected root through complete, + static tool or handoff edges; - every extracted observation has a complete canonical identity, and no selector or reviewed cross-source binding is ambiguous or conflicting; - its effect is established by a reviewed manifest declaration or a structural @@ -31,10 +34,17 @@ suppressed, baseline-matched, waived through a severity override, or converted to known evidence by `--no-heuristics` or `human_ack`. Machine consumers should inspect -`release_decision.evidence_coverage.semantic_coverage` and -`identity_coverage`, then work `evidence_gaps[]` in order. Packet schema v0.9 -mirrors this contract, while capability standard v0.3 carries the same -normalized assessment in capability lock v0.4 and lock-diff v0.5 artifacts. +`release_decision.evidence_coverage.semantic_coverage`, `binding_coverage`, and +`identity_coverage`, then work `evidence_gaps[]` in order. Packet schema v0.10 +mirrors this contract, while capability standard v0.4 carries the same +normalized assessment and binding hash in capability lock v0.5 and lock-diff +v0.6 artifacts. + +Catalog membership never implies binding. `tool_catalog[]` contains every +canonical extracted declaration; `tool_inventory[]`, actions, checks, and +capability facts contain only tools proven reachable from the root. Reviewed +closed-world declarations live under `agent_bindings`; coding agents must not +invent or auto-apply them. The release decision also carries an explicit machine boundary: diff --git a/docs/report-reading-for-agents.md b/docs/report-reading-for-agents.md index 51cffc23..a610ac08 100644 --- a/docs/report-reading-for-agents.md +++ b/docs/report-reading-for-agents.md @@ -146,7 +146,7 @@ Alongside `report.json`, scan emits a reviewer-shaped Release Evidence Packet at - §1 verdict — derives from `release_decision.decision` only. Never derive a verdict from `summary.status`. - §10 ("What this packet did NOT prove") — always lists prompt robustness, runtime behavior, model correctness, adversarial resistance. -The packet schema is `0.8`; full schema at [`docs/packet-schema.v0.9.json`](packet-schema.v0.9.json). It projects report v0.30 semantic coverage and gap remediation; v0.7 is a frozen reference. +The packet schema is `0.10`; full schema at [`docs/packet-schema.v0.10.json`](packet-schema.v0.10.json). It projects report v0.31 binding and semantic coverage plus gap remediation; v0.9 is a frozen reference. --- @@ -227,8 +227,8 @@ Surface the `next_action` to the user rather than scraping prose. The full diagn | Schema | Current | Frozen references | File | |---|---|---|---| -| Report | `0.29` | `0.28`, `0.27`, `0.26`, `0.25`, `0.24`, `0.23`, `0.22`, `0.21`, `0.20`, `0.19`, `0.18`, `0.17`, `0.16`, `0.15`, `0.14`, `0.13`, `0.12`, `0.11`, `0.10`, `0.9`, `0.8`, `0.7`, `0.6`, `0.5`, `0.4`, `0.3`, `0.2`, `0.1` | [`report-schema.v0.30.json`](report-schema.v0.30.json) | -| Packet | `0.8` | `0.7`, `0.6`, `0.5`, `0.4`, `0.3`, `0.2`, `0.1` | [`packet-schema.v0.9.json`](packet-schema.v0.9.json) | +| Report | `0.31` | `0.30`, `0.29`, `0.28`, `0.27`, `0.26`, `0.25`, `0.24`, `0.23`, `0.22`, `0.21`, `0.20`, `0.19`, `0.18`, `0.17`, `0.16`, `0.15`, `0.14`, `0.13`, `0.12`, `0.11`, `0.10`, `0.9`, `0.8`, `0.7`, `0.6`, `0.5`, `0.4`, `0.3`, `0.2`, `0.1` | [`report-schema.v0.31.json`](report-schema.v0.31.json) | +| Packet | `0.10` | `0.9`, `0.8`, `0.7`, `0.6`, `0.5`, `0.4`, `0.3`, `0.2`, `0.1` | [`packet-schema.v0.10.json`](packet-schema.v0.10.json) | | Manifest | `0.1` | — | [`manifest-v0.1.json`](manifest-v0.1.json) | | CLI contract | `5` | — | `agents-shipgate contract --json` | diff --git a/docs/report-schema.v0.31.json b/docs/report-schema.v0.31.json new file mode 100644 index 00000000..d1fe6c87 --- /dev/null +++ b/docs/report-schema.v0.31.json @@ -0,0 +1,7096 @@ +{ + "$defs": { + "ActionApprovalFact": { + "additionalProperties": false, + "properties": { + "required": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Required" + }, + "threshold": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Threshold" + } + }, + "required": [ + "required", + "threshold" + ], + "title": "ActionApprovalFact", + "type": "object" + }, + "ActionEvidenceFact": { + "additionalProperties": false, + "properties": { + "approval_ticket": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Approval Ticket" + }, + "owner": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Owner" + }, + "runbook": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Runbook" + } + }, + "required": [ + "approval_ticket", + "owner", + "runbook" + ], + "title": "ActionEvidenceFact", + "type": "object" + }, + "ActionFact": { + "additionalProperties": false, + "properties": { + "action_id": { + "title": "Action Id", + "type": "string" + }, + "agent_id": { + "title": "Agent Id", + "type": "string" + }, + "approval_policy": { + "$ref": "#/$defs/ActionApprovalFact" + }, + "effect": { + "enum": [ + "read", + "write", + "destructive", + "external_communication", + "financial_write", + "production_operation", + "privileged_data_access", + "code_execution", + "identity_access" + ], + "title": "Effect", + "type": "string" + }, + "evidence": { + "$ref": "#/$defs/ActionEvidenceFact" + }, + "hashes": { + "$ref": "#/$defs/ActionSurfaceHashes" + }, + "input_fields": { + "items": { + "type": "string" + }, + "title": "Input Fields", + "type": "array" + }, + "input_schema_hash": { + "title": "Input Schema Hash", + "type": "string" + }, + "operation": { + "title": "Operation", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "required_input_fields": { + "items": { + "type": "string" + }, + "title": "Required Input Fields", + "type": "array" + }, + "required_scopes": { + "items": { + "type": "string" + }, + "title": "Required Scopes", + "type": "array" + }, + "risk_tags": { + "items": { + "type": "string" + }, + "title": "Risk Tags", + "type": "array" + }, + "safeguards": { + "$ref": "#/$defs/ActionSafeguardsFact" + }, + "semantic_assessment": { + "anyOf": [ + { + "$ref": "#/$defs/ToolSemanticEvidence" + }, + { + "type": "null" + } + ], + "default": null + }, + "source_end_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source End Line" + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + }, + "source_location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Location" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_start_column": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Column" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "source_type": { + "title": "Source Type", + "type": "string" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "action_id", + "agent_id", + "approval_policy", + "effect", + "evidence", + "hashes", + "input_fields", + "input_schema_hash", + "operation", + "provider", + "required_input_fields", + "required_scopes", + "risk_tags", + "safeguards", + "source_id", + "source_type", + "tool_id", + "tool_name" + ], + "title": "ActionFact", + "type": "object" + }, + "ActionSafeguardsFact": { + "additionalProperties": false, + "properties": { + "audit_log": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audit Log" + }, + "dry_run": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dry Run" + }, + "idempotency": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Idempotency" + }, + "rollback": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rollback" + } + }, + "required": [ + "audit_log", + "dry_run", + "idempotency", + "rollback" + ], + "title": "ActionSafeguardsFact", + "type": "object" + }, + "ActionSurfaceChange": { + "additionalProperties": false, + "properties": { + "action_id": { + "title": "Action Id", + "type": "string" + }, + "added": { + "items": { + "type": "string" + }, + "title": "Added", + "type": "array" + }, + "after": { + "default": null, + "title": "After" + }, + "agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Agent Id" + }, + "before": { + "default": null, + "title": "Before" + }, + "operation": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Operation" + }, + "reason": { + "title": "Reason", + "type": "string" + }, + "removed": { + "items": { + "type": "string" + }, + "title": "Removed", + "type": "array" + }, + "severity": { + "default": "info", + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Severity", + "type": "string" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + }, + "tool_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Name" + }, + "type": { + "enum": [ + "ACTION_ADDED", + "ACTION_REMOVED", + "ACTION_MODIFIED", + "SCOPE_EXPANDED", + "EFFECT_ESCALATED", + "RISK_TAG_ADDED", + "APPROVAL_REMOVED", + "SAFEGUARD_REMOVED", + "INPUT_SCHEMA_EXPANDED" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "action_id", + "added", + "after", + "agent_id", + "before", + "operation", + "reason", + "removed", + "severity", + "tool_name", + "type" + ], + "title": "ActionSurfaceChange", + "type": "object" + }, + "ActionSurfaceDiff": { + "additionalProperties": false, + "properties": { + "added": { + "items": { + "$ref": "#/$defs/ActionSurfaceChange" + }, + "title": "Added", + "type": "array" + }, + "base": { + "$ref": "#/$defs/ToolSurfaceDiffBase" + }, + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "modified": { + "items": { + "$ref": "#/$defs/ActionSurfaceChange" + }, + "title": "Modified", + "type": "array" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "removed": { + "items": { + "$ref": "#/$defs/ActionSurfaceChange" + }, + "title": "Removed", + "type": "array" + }, + "summary": { + "$ref": "#/$defs/ActionSurfaceDiffSummary" + } + }, + "required": [ + "added", + "base", + "enabled", + "modified", + "notes", + "removed", + "summary" + ], + "title": "ActionSurfaceDiff", + "type": "object" + }, + "ActionSurfaceDiffSummary": { + "additionalProperties": false, + "properties": { + "actions_added": { + "default": 0, + "title": "Actions Added", + "type": "integer" + }, + "actions_modified": { + "default": 0, + "title": "Actions Modified", + "type": "integer" + }, + "actions_removed": { + "default": 0, + "title": "Actions Removed", + "type": "integer" + }, + "approvals_removed": { + "default": 0, + "title": "Approvals Removed", + "type": "integer" + }, + "blocking_findings": { + "default": 0, + "title": "Blocking Findings", + "type": "integer" + }, + "effect_escalations": { + "default": 0, + "title": "Effect Escalations", + "type": "integer" + }, + "input_schema_expansions": { + "default": 0, + "title": "Input Schema Expansions", + "type": "integer" + }, + "risk_tags_added": { + "default": 0, + "title": "Risk Tags Added", + "type": "integer" + }, + "safeguards_removed": { + "default": 0, + "title": "Safeguards Removed", + "type": "integer" + }, + "scope_expansions": { + "default": 0, + "title": "Scope Expansions", + "type": "integer" + } + }, + "required": [ + "actions_added", + "actions_modified", + "actions_removed", + "approvals_removed", + "blocking_findings", + "effect_escalations", + "input_schema_expansions", + "risk_tags_added", + "safeguards_removed", + "scope_expansions" + ], + "title": "ActionSurfaceDiffSummary", + "type": "object" + }, + "ActionSurfaceFacts": { + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "$ref": "#/$defs/ActionFact" + }, + "title": "Actions", + "type": "array" + }, + "snapshot_version": { + "default": "0.3", + "title": "Snapshot Version", + "type": "string" + } + }, + "required": [ + "actions", + "snapshot_version" + ], + "title": "ActionSurfaceFacts", + "type": "object" + }, + "ActionSurfaceHashes": { + "additionalProperties": false, + "properties": { + "identity_hash": { + "title": "Identity Hash", + "type": "string" + }, + "policy_hash": { + "title": "Policy Hash", + "type": "string" + }, + "risk_hash": { + "title": "Risk Hash", + "type": "string" + }, + "schema_hash": { + "title": "Schema Hash", + "type": "string" + } + }, + "required": [ + "identity_hash", + "policy_hash", + "risk_hash", + "schema_hash" + ], + "title": "ActionSurfaceHashes", + "type": "object" + }, + "AgentBindingGraphAssessment": { + "additionalProperties": false, + "properties": { + "agents": { + "items": { + "$ref": "#/$defs/AgentBindingNode" + }, + "title": "Agents", + "type": "array" + }, + "handoff_edges": { + "items": { + "$ref": "#/$defs/AgentHandoffBindingEdge" + }, + "title": "Handoff Edges", + "type": "array" + }, + "issues": { + "items": { + "$ref": "#/$defs/AgentBindingIssue" + }, + "title": "Issues", + "type": "array" + }, + "pass_eligible": { + "default": false, + "title": "Pass Eligible", + "type": "boolean" + }, + "possible_tool_ids": { + "items": { + "type": "string" + }, + "title": "Possible Tool Ids", + "type": "array" + }, + "reachable_tool_ids": { + "items": { + "type": "string" + }, + "title": "Reachable Tool Ids", + "type": "array" + }, + "root_agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Root Agent Id" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + }, + "tool_edges": { + "items": { + "$ref": "#/$defs/AgentToolBindingEdge" + }, + "title": "Tool Edges", + "type": "array" + }, + "unbound_tool_ids": { + "items": { + "type": "string" + }, + "title": "Unbound Tool Ids", + "type": "array" + } + }, + "required": [ + "status" + ], + "title": "AgentBindingGraphAssessment", + "type": "object" + }, + "AgentBindingIssue": { + "additionalProperties": false, + "properties": { + "agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Agent Id" + }, + "kind": { + "enum": [ + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation" + ], + "title": "Kind", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "kind", + "message" + ], + "title": "AgentBindingIssue", + "type": "object" + }, + "AgentBindingNode": { + "additionalProperties": false, + "properties": { + "agent_id": { + "title": "Agent Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + } + }, + "required": [ + "agent_id", + "name" + ], + "title": "AgentBindingNode", + "type": "object" + }, + "AgentHandoffBindingEdge": { + "additionalProperties": false, + "properties": { + "complete": { + "default": true, + "title": "Complete", + "type": "boolean" + }, + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "edge_type": { + "enum": [ + "subagent", + "handoff" + ], + "title": "Edge Type", + "type": "string" + }, + "provenance_kind": { + "enum": [ + "static_declaration", + "ast_extraction", + "keyword_heuristic", + "regex_heuristic", + "policy_pack", + "runtime_trace" + ], + "title": "Provenance Kind", + "type": "string" + }, + "source": { + "title": "Source", + "type": "string" + }, + "source_agent_id": { + "title": "Source Agent Id", + "type": "string" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "target_agent_id": { + "title": "Target Agent Id", + "type": "string" + } + }, + "required": [ + "source_agent_id", + "target_agent_id", + "edge_type", + "confidence", + "provenance_kind", + "source" + ], + "title": "AgentHandoffBindingEdge", + "type": "object" + }, + "AgentSummary": { + "additionalProperties": false, + "description": "Top-level summary block shaped for one-fetch agent consumption.\n\nDeterministic projection of (``release_decision``, ``findings[].agent_action``).\nA coding agent that wants the headline numbers can read this block\ninstead of traversing arrays. All fields are derived; this block\ncannot disagree with the underlying data.", + "properties": { + "auto_appliable_patches": { + "default": 0, + "title": "Auto Appliable Patches", + "type": "integer" + }, + "blocker_count": { + "default": 0, + "title": "Blocker Count", + "type": "integer" + }, + "first_recommended_action": { + "anyOf": [ + { + "$ref": "#/$defs/AgentSummaryAction" + }, + { + "type": "null" + } + ], + "default": null + }, + "headline": { + "title": "Headline", + "type": "string" + }, + "needs_human_review": { + "default": 0, + "title": "Needs Human Review", + "type": "integer" + }, + "review_item_count": { + "default": 0, + "title": "Review Item Count", + "type": "integer" + }, + "verdict": { + "enum": [ + "blocked", + "review_required", + "insufficient_evidence", + "passed" + ], + "title": "Verdict", + "type": "string" + } + }, + "required": [ + "auto_appliable_patches", + "blocker_count", + "first_recommended_action", + "headline", + "needs_human_review", + "review_item_count", + "verdict" + ], + "title": "AgentSummary", + "type": "object" + }, + "AgentSummaryAction": { + "additionalProperties": false, + "description": "A single recommended next step shaped for direct agent consumption.\n\nMirrors the ``next_actions[]`` shape used elsewhere in the contract\n(kind/command/why) so callers that already handle diagnostic\nnext_actions can reuse the same renderer here.", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "kind": { + "default": "command", + "enum": [ + "command", + "info" + ], + "title": "Kind", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "command", + "kind", + "why" + ], + "title": "AgentSummaryAction", + "type": "object" + }, + "AgentToolBindingEdge": { + "additionalProperties": false, + "properties": { + "agent_id": { + "title": "Agent Id", + "type": "string" + }, + "complete": { + "default": true, + "title": "Complete", + "type": "boolean" + }, + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "edge_type": { + "enum": [ + "direct_tool", + "tool_node", + "toolset", + "workflow", + "subagent", + "handoff" + ], + "title": "Edge Type", + "type": "string" + }, + "provenance_kind": { + "enum": [ + "static_declaration", + "ast_extraction", + "keyword_heuristic", + "regex_heuristic", + "policy_pack", + "runtime_trace" + ], + "title": "Provenance Kind", + "type": "string" + }, + "source": { + "title": "Source", + "type": "string" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + } + }, + "required": [ + "agent_id", + "tool_id", + "edge_type", + "confidence", + "provenance_kind", + "source" + ], + "title": "AgentToolBindingEdge", + "type": "object" + }, + "AppendPointerPatch": { + "additionalProperties": false, + "description": "Append a value to the list at a JSON pointer.", + "properties": { + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "kind": { + "const": "append_pointer", + "default": "append_pointer", + "title": "Kind", + "type": "string" + }, + "pointer": { + "title": "Pointer", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "target_file": { + "title": "Target File", + "type": "string" + }, + "target_format": { + "enum": [ + "yaml", + "json" + ], + "title": "Target Format", + "type": "string" + }, + "target_sha256": { + "title": "Target Sha256", + "type": "string" + }, + "value": { + "title": "Value" + } + }, + "required": [ + "target_file", + "pointer", + "value", + "target_format", + "confidence", + "rationale", + "target_sha256" + ], + "title": "AppendPointerPatch", + "type": "object" + }, + "AuthoritySemanticEvidence": { + "additionalProperties": false, + "properties": { + "auth_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auth Type" + }, + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "credential_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Credential Mode" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "mode": { + "enum": [ + "none", + "scoped", + "unscoped", + "ambient", + "unknown" + ], + "title": "Mode", + "type": "string" + }, + "scopes": { + "items": { + "type": "string" + }, + "title": "Scopes", + "type": "array" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "mode" + ], + "title": "AuthoritySemanticEvidence", + "type": "object" + }, + "BaselineDelta": { + "properties": { + "enabled": { + "title": "Enabled", + "type": "boolean" + }, + "matched_count": { + "default": 0, + "title": "Matched Count", + "type": "integer" + }, + "new_count": { + "default": 0, + "title": "New Count", + "type": "integer" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "resolved_count": { + "default": 0, + "title": "Resolved Count", + "type": "integer" + } + }, + "required": [ + "enabled", + "matched_count", + "new_count", + "resolved_count" + ], + "title": "BaselineDelta", + "type": "object" + }, + "BaselineSummary": { + "properties": { + "matched_count": { + "default": 0, + "title": "Matched Count", + "type": "integer" + }, + "new_count": { + "default": 0, + "title": "New Count", + "type": "integer" + }, + "path": { + "title": "Path", + "type": "string" + }, + "resolved_count": { + "default": 0, + "title": "Resolved Count", + "type": "integer" + } + }, + "required": [ + "path" + ], + "title": "BaselineSummary", + "type": "object" + }, + "BindingCoverageDecision": { + "properties": { + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible": { + "default": false, + "title": "Pass Eligible", + "type": "boolean" + }, + "possible_tools": { + "default": 0, + "title": "Possible Tools", + "type": "integer" + }, + "reachable_tools": { + "default": 0, + "title": "Reachable Tools", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "total_catalog_tools": { + "default": 0, + "title": "Total Catalog Tools", + "type": "integer" + }, + "unbound_tools": { + "default": 0, + "title": "Unbound Tools", + "type": "integer" + } + }, + "title": "BindingCoverageDecision", + "type": "object" + }, + "BindingSemanticEvidence": { + "additionalProperties": false, + "properties": { + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "pass_eligible": { + "default": false, + "title": "Pass Eligible", + "type": "boolean" + }, + "reachable_path": { + "items": { + "type": "string" + }, + "title": "Reachable Path", + "type": "array" + }, + "root_agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Root Agent Id" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "confidence" + ], + "title": "BindingSemanticEvidence", + "type": "object" + }, + "BindingSurfaceDiff": { + "additionalProperties": false, + "properties": { + "added_handoffs": { + "items": { + "type": "string" + }, + "title": "Added Handoffs", + "type": "array" + }, + "added_reachable_tool_ids": { + "items": { + "type": "string" + }, + "title": "Added Reachable Tool Ids", + "type": "array" + }, + "base_report_schema_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Report Schema Version" + }, + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "removed_handoffs": { + "items": { + "type": "string" + }, + "title": "Removed Handoffs", + "type": "array" + }, + "removed_reachable_tool_ids": { + "items": { + "type": "string" + }, + "title": "Removed Reachable Tool Ids", + "type": "array" + } + }, + "title": "BindingSurfaceDiff", + "type": "object" + }, + "CapabilityChangeBlock": { + "additionalProperties": false, + "description": "The diff-derived capability delta, grouped by direction.\n\nReviewer-facing projection over ``action_surface_diff`` /\n``tool_surface_diff`` (roadmap \u00a77.1). Four member lists \u2014\n``added`` / ``removed`` / ``broadened`` / ``narrowed`` \u2014 plus\n``enabled`` (mirrors the surface-diff enabled flag: ``False`` when no\nbase is available, so the block is a stable empty shape rather than\nabsent). Never gates on its own.", + "properties": { + "added": { + "items": { + "$ref": "#/$defs/CapabilityChangeMember" + }, + "title": "Added", + "type": "array" + }, + "broadened": { + "items": { + "$ref": "#/$defs/CapabilityChangeMember" + }, + "title": "Broadened", + "type": "array" + }, + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "narrowed": { + "items": { + "$ref": "#/$defs/CapabilityChangeMember" + }, + "title": "Narrowed", + "type": "array" + }, + "removed": { + "items": { + "$ref": "#/$defs/CapabilityChangeMember" + }, + "title": "Removed", + "type": "array" + } + }, + "required": [ + "added", + "broadened", + "enabled", + "narrowed", + "removed" + ], + "title": "CapabilityChangeBlock", + "type": "object" + }, + "CapabilityChangeMember": { + "additionalProperties": false, + "description": "One capability member in a base\u2192head delta.\n\nCarries enough to identify the capability \u2014 ``tool`` plus the\n``action`` and ``scope`` it concerns \u2014 and, for ``broadened`` /\n``narrowed`` directions, the ``before_scope`` / ``after_scope`` so a\nreviewer can see exactly how the scope moved. Membership changes\n(``added`` / ``removed``) leave the before/after scope fields ``None``.\n\nA deterministic ``id`` supplied by the capability-change builder keeps\nthe member stable across runs for byte-equivalent output.", + "properties": { + "action": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Action" + }, + "after_capability_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "After Capability Id" + }, + "after_scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "After Scope" + }, + "before_capability_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Before Capability Id" + }, + "before_scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Before Scope" + }, + "changed_hashes": { + "items": { + "enum": [ + "identity_hash", + "binding_hash", + "effect_hash", + "authority_hash", + "control_hash", + "schema_hash", + "risk_hash", + "evidence_hash" + ], + "type": "string" + }, + "title": "Changed Hashes", + "type": "array" + }, + "confidence": { + "default": "medium", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "direction": { + "enum": [ + "added", + "removed", + "broadened", + "narrowed" + ], + "title": "Direction", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "provenance_kind": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provenance Kind" + }, + "rationale": { + "default": "", + "title": "Rationale", + "type": "string" + }, + "related_finding_ids": { + "items": { + "type": "string" + }, + "title": "Related Finding Ids", + "type": "array" + }, + "release_impact": { + "default": "informational", + "enum": [ + "none", + "informational", + "review_required", + "blocks_release", + "insufficient_evidence" + ], + "title": "Release Impact", + "type": "string" + }, + "risk_tags": { + "items": { + "type": "string" + }, + "title": "Risk Tags", + "type": "array" + }, + "scope": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Scope" + }, + "semantic_changes": { + "items": { + "$ref": "#/$defs/CapabilitySemanticChange" + }, + "title": "Semantic Changes", + "type": "array" + }, + "semantic_direction": { + "default": "unknown", + "enum": [ + "added", + "removed", + "broadened", + "narrowed", + "mixed", + "unknown", + "evidence_only" + ], + "title": "Semantic Direction", + "type": "string" + }, + "subject_kind": { + "default": "unknown", + "enum": [ + "tool", + "action", + "scope", + "policy", + "ci", + "baseline", + "agent_instruction", + "manifest", + "unknown" + ], + "title": "Subject Kind", + "type": "string" + }, + "tool": { + "title": "Tool", + "type": "string" + } + }, + "required": [ + "action", + "after_scope", + "before_scope", + "confidence", + "direction", + "id", + "provenance_kind", + "rationale", + "related_finding_ids", + "release_impact", + "risk_tags", + "scope", + "subject_kind", + "tool" + ], + "title": "CapabilityChangeMember", + "type": "object" + }, + "CapabilityFact": { + "properties": { + "auth_scopes": { + "items": { + "type": "string" + }, + "title": "Auth Scopes", + "type": "array" + }, + "capability": { + "title": "Capability", + "type": "string" + }, + "control_status": { + "enum": [ + "missing", + "partial", + "present", + "unknown" + ], + "title": "Control Status", + "type": "string" + }, + "effect": { + "anyOf": [ + { + "enum": [ + "read", + "write", + "destructive", + "external_communication", + "financial_write", + "production_operation", + "privileged_data_access", + "code_execution", + "identity_access" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Effect" + }, + "id": { + "title": "Id", + "type": "string" + }, + "included_reason": { + "enum": [ + "high_risk_tag", + "wildcard_exposure", + "referenced_by_critical_finding", + "referenced_by_high_finding", + "referenced_by_medium_finding" + ], + "title": "Included Reason", + "type": "string" + }, + "owner": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Owner" + }, + "related_findings": { + "items": { + "type": "string" + }, + "title": "Related Findings", + "type": "array" + }, + "risk_tags": { + "items": { + "type": "string" + }, + "title": "Risk Tags", + "type": "array" + }, + "semantic_assessment": { + "anyOf": [ + { + "$ref": "#/$defs/ToolSemanticEvidence" + }, + { + "type": "null" + } + ], + "default": null + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_type": { + "title": "Source Type", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + }, + "tool_name": { + "title": "Tool Name", + "type": "string" + } + }, + "required": [ + "auth_scopes", + "capability", + "control_status", + "id", + "included_reason", + "owner", + "related_findings", + "risk_tags", + "source_ref", + "source_type", + "tool_name" + ], + "title": "CapabilityFact", + "type": "object" + }, + "CapabilityPolicyEvidence": { + "additionalProperties": false, + "description": "Capability-level audit evidence for a policy match.\n\nThis is explanatory metadata only. It lets reviewers see which durable\ncapability fact matched a policy rule without folding that metadata into\nthe legacy ``Finding.evidence`` fingerprint input.", + "properties": { + "authority": { + "additionalProperties": true, + "title": "Authority", + "type": "object" + }, + "capability_id": { + "title": "Capability Id", + "type": "string" + }, + "controls": { + "additionalProperties": true, + "title": "Controls", + "type": "object" + }, + "effect": { + "additionalProperties": true, + "title": "Effect", + "type": "object" + }, + "hashes": { + "additionalProperties": { + "type": "string" + }, + "title": "Hashes", + "type": "object" + }, + "identity": { + "additionalProperties": true, + "title": "Identity", + "type": "object" + }, + "matched_predicates": { + "additionalProperties": true, + "title": "Matched Predicates", + "type": "object" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "capability_id", + "identity", + "effect", + "authority", + "controls", + "hashes" + ], + "title": "CapabilityPolicyEvidence", + "type": "object" + }, + "CapabilityRuntimeEvidence": { + "additionalProperties": false, + "description": "Top-level audit block for declared local runtime trace artifacts.", + "properties": { + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "matched": { + "items": { + "$ref": "#/$defs/CapabilityTraceEvidenceV1" + }, + "title": "Matched", + "type": "array" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "source_provenance": { + "items": { + "$ref": "#/$defs/SourceReference" + }, + "title": "Source Provenance", + "type": "array" + }, + "summary": { + "$ref": "#/$defs/CapabilityTraceEvidenceSummary" + }, + "unmatched": { + "items": { + "$ref": "#/$defs/CapabilityTraceEvidenceV1" + }, + "title": "Unmatched", + "type": "array" + } + }, + "title": "CapabilityRuntimeEvidence", + "type": "object" + }, + "CapabilitySemanticChange": { + "additionalProperties": false, + "description": "One semantic explanation for a capability delta.", + "properties": { + "after": { + "default": null, + "title": "After" + }, + "before": { + "default": null, + "title": "Before" + }, + "field": { + "title": "Field", + "type": "string" + }, + "kind": { + "title": "Kind", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + } + }, + "required": [ + "kind", + "field", + "rationale" + ], + "title": "CapabilitySemanticChange", + "type": "object" + }, + "CapabilityTraceEvidenceSummary": { + "additionalProperties": false, + "description": "Deterministic counts for opt-in local runtime trace evidence.", + "properties": { + "agent_trace_count": { + "default": 0, + "title": "Agent Trace Count", + "type": "integer" + }, + "api_trace_count": { + "default": 0, + "title": "Api Trace Count", + "type": "integer" + }, + "approval_trace_count": { + "default": 0, + "title": "Approval Trace Count", + "type": "integer" + }, + "matched_trace_count": { + "default": 0, + "title": "Matched Trace Count", + "type": "integer" + }, + "source_count": { + "default": 0, + "title": "Source Count", + "type": "integer" + }, + "trace_count": { + "default": 0, + "title": "Trace Count", + "type": "integer" + }, + "unmatched_trace_count": { + "default": 0, + "title": "Unmatched Trace Count", + "type": "integer" + }, + "warning_count": { + "default": 0, + "title": "Warning Count", + "type": "integer" + } + }, + "title": "CapabilityTraceEvidenceSummary", + "type": "object" + }, + "CapabilityTraceEvidenceV1": { + "additionalProperties": false, + "description": "Allowlisted local trace event linked to a durable capability fact.\n\nRaw prompts, messages, tool arguments, outputs, and arbitrary payloads\nare intentionally absent. ``observed`` contains only normalized scalar\nevidence from the allowlist enforced by ``inputs.traces``.", + "properties": { + "capability_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Capability Id" + }, + "event_hash": { + "title": "Event Hash", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "match_reason": { + "enum": [ + "capability_id", + "tool_name", + "unknown_tool", + "ambiguous_tool", + "invalid_capability_id", + "missing_tool_name" + ], + "title": "Match Reason", + "type": "string" + }, + "matched": { + "default": false, + "title": "Matched", + "type": "boolean" + }, + "matched_capability_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Matched Capability Id" + }, + "observed": { + "additionalProperties": true, + "title": "Observed", + "type": "object" + }, + "operation": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Operation" + }, + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "source_hash": { + "title": "Source Hash", + "type": "string" + }, + "source_type": { + "title": "Source Type", + "type": "string" + }, + "tool_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Name" + } + }, + "required": [ + "id", + "source_type", + "match_reason", + "event_hash", + "source_hash" + ], + "title": "CapabilityTraceEvidenceV1", + "type": "object" + }, + "CodexPluginAppSummary": { + "additionalProperties": true, + "properties": { + "connector_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Connector Id" + }, + "location": { + "$ref": "#/$defs/CodexPluginSourceLocation" + }, + "name": { + "title": "Name", + "type": "string" + }, + "path": { + "title": "Path", + "type": "string" + }, + "plugin": { + "title": "Plugin", + "type": "string" + } + }, + "required": [ + "plugin", + "name", + "path" + ], + "title": "CodexPluginAppSummary", + "type": "object" + }, + "CodexPluginComponentPathIssue": { + "additionalProperties": true, + "properties": { + "component": { + "title": "Component", + "type": "string" + }, + "path": { + "title": "Path", + "type": "string" + }, + "plugin": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Plugin" + }, + "reason": { + "title": "Reason", + "type": "string" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + } + }, + "required": [ + "component", + "path", + "reason" + ], + "title": "CodexPluginComponentPathIssue", + "type": "object" + }, + "CodexPluginHookStub": { + "additionalProperties": true, + "properties": { + "command": { + "title": "Command", + "type": "string" + }, + "confidence": { + "default": "medium", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "location": { + "$ref": "#/$defs/CodexPluginSourceLocation" + }, + "name": { + "title": "Name", + "type": "string" + }, + "path": { + "title": "Path", + "type": "string" + }, + "plugin": { + "title": "Plugin", + "type": "string" + }, + "risk_tags": { + "items": { + "type": "string" + }, + "title": "Risk Tags", + "type": "array" + } + }, + "required": [ + "plugin", + "name", + "command", + "path" + ], + "title": "CodexPluginHookStub", + "type": "object" + }, + "CodexPluginMarketplaceSummary": { + "additionalProperties": true, + "properties": { + "missing_policy_entries": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Missing Policy Entries", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "path": { + "title": "Path", + "type": "string" + }, + "plugin_count": { + "default": 0, + "title": "Plugin Count", + "type": "integer" + }, + "skipped_entries": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Skipped Entries", + "type": "array" + }, + "source_id": { + "title": "Source Id", + "type": "string" + } + }, + "required": [ + "source_id", + "path" + ], + "title": "CodexPluginMarketplaceSummary", + "type": "object" + }, + "CodexPluginMcpServerStub": { + "additionalProperties": true, + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "inventory_loaded": { + "default": false, + "title": "Inventory Loaded", + "type": "boolean" + }, + "inventory_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Inventory Path" + }, + "location": { + "$ref": "#/$defs/CodexPluginSourceLocation" + }, + "path": { + "title": "Path", + "type": "string" + }, + "plugin": { + "title": "Plugin", + "type": "string" + }, + "server": { + "title": "Server", + "type": "string" + } + }, + "required": [ + "plugin", + "server", + "path" + ], + "title": "CodexPluginMcpServerStub", + "type": "object" + }, + "CodexPluginSkillSummary": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "duplicate": { + "default": false, + "title": "Duplicate", + "type": "boolean" + }, + "location": { + "$ref": "#/$defs/CodexPluginSourceLocation" + }, + "missing_fields": { + "items": { + "type": "string" + }, + "title": "Missing Fields", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "path": { + "title": "Path", + "type": "string" + }, + "plugin": { + "title": "Plugin", + "type": "string" + } + }, + "required": [ + "plugin", + "path" + ], + "title": "CodexPluginSkillSummary", + "type": "object" + }, + "CodexPluginSourceLocation": { + "additionalProperties": false, + "properties": { + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_start_column": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Column" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + } + }, + "title": "CodexPluginSourceLocation", + "type": "object" + }, + "CodexPluginSummary": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "duplicate_name": { + "default": false, + "title": "Duplicate Name", + "type": "boolean" + }, + "duplicate_root": { + "default": false, + "title": "Duplicate Root", + "type": "boolean" + }, + "location": { + "$ref": "#/$defs/CodexPluginSourceLocation" + }, + "manifest_path": { + "title": "Manifest Path", + "type": "string" + }, + "marketplace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Marketplace" + }, + "missing_fields": { + "items": { + "type": "string" + }, + "title": "Missing Fields", + "type": "array" + }, + "name": { + "title": "Name", + "type": "string" + }, + "name_mismatch": { + "default": false, + "title": "Name Mismatch", + "type": "boolean" + }, + "root_path": { + "title": "Root Path", + "type": "string" + }, + "source_id": { + "title": "Source Id", + "type": "string" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Version" + } + }, + "required": [ + "source_id", + "name", + "root_path", + "manifest_path" + ], + "title": "CodexPluginSummary", + "type": "object" + }, + "CodexPluginSurface": { + "additionalProperties": true, + "properties": { + "app_count": { + "default": 0, + "title": "App Count", + "type": "integer" + }, + "apps": { + "items": { + "$ref": "#/$defs/CodexPluginAppSummary" + }, + "title": "Apps", + "type": "array" + }, + "component_path_issues": { + "items": { + "$ref": "#/$defs/CodexPluginComponentPathIssue" + }, + "title": "Component Path Issues", + "type": "array" + }, + "hook_stub_count": { + "default": 0, + "title": "Hook Stub Count", + "type": "integer" + }, + "hook_stubs": { + "items": { + "$ref": "#/$defs/CodexPluginHookStub" + }, + "title": "Hook Stubs", + "type": "array" + }, + "marketplace_count": { + "default": 0, + "title": "Marketplace Count", + "type": "integer" + }, + "marketplaces": { + "items": { + "$ref": "#/$defs/CodexPluginMarketplaceSummary" + }, + "title": "Marketplaces", + "type": "array" + }, + "mcp_inventory_file_count": { + "default": 0, + "title": "Mcp Inventory File Count", + "type": "integer" + }, + "mcp_inventory_files": { + "items": { + "type": "string" + }, + "title": "Mcp Inventory Files", + "type": "array" + }, + "mcp_server_stub_count": { + "default": 0, + "title": "Mcp Server Stub Count", + "type": "integer" + }, + "mcp_server_stubs": { + "items": { + "$ref": "#/$defs/CodexPluginMcpServerStub" + }, + "title": "Mcp Server Stubs", + "type": "array" + }, + "plugin_count": { + "default": 0, + "title": "Plugin Count", + "type": "integer" + }, + "plugins": { + "items": { + "$ref": "#/$defs/CodexPluginSummary" + }, + "title": "Plugins", + "type": "array" + }, + "skill_count": { + "default": 0, + "title": "Skill Count", + "type": "integer" + }, + "skills": { + "items": { + "$ref": "#/$defs/CodexPluginSkillSummary" + }, + "title": "Skills", + "type": "array" + }, + "warnings": { + "items": { + "type": "string" + }, + "title": "Warnings", + "type": "array" + } + }, + "title": "CodexPluginSurface", + "type": "object" + }, + "ContributionRule": { + "additionalProperties": false, + "description": "Per-finding audit row explaining how a finding contributed to the\nrelease decision.\n\nAdditive in v0.17. Every finding in `report.findings` produces\nexactly one ContributionRule. Reading the contribution rule is\nsufficient to predict the gate outcome for that finding without\nre-deriving the decision logic; the set of valid `(rule, category)`\npairs is the contract documented in STABILITY.md \"Release decision\ntruth table\".", + "properties": { + "category": { + "enum": [ + "blocker", + "review_item", + "excluded" + ], + "title": "Category", + "type": "string" + }, + "check_id": { + "title": "Check Id", + "type": "string" + }, + "finding_id": { + "title": "Finding Id", + "type": "string" + }, + "fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fingerprint" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "rule": { + "enum": [ + "policy_block_new", + "severity_block_new", + "policy_baseline_accepted", + "severity_baseline_accepted", + "review_required", + "sub_threshold", + "suppressed" + ], + "title": "Rule", + "type": "string" + } + }, + "required": [ + "category", + "check_id", + "finding_id", + "fingerprint", + "rationale", + "rule" + ], + "title": "ContributionRule", + "type": "object" + }, + "DeclaredIntention": { + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "intent_tags": { + "items": { + "type": "string" + }, + "title": "Intent Tags", + "type": "array" + }, + "kind": { + "enum": [ + "declared_purpose", + "prohibited_action", + "instruction_preview" + ], + "title": "Kind", + "type": "string" + }, + "source": { + "title": "Source", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + } + }, + "required": [ + "id", + "intent_tags", + "kind", + "source", + "text" + ], + "title": "DeclaredIntention", + "type": "object" + }, + "EffectSemanticEvidence": { + "additionalProperties": false, + "properties": { + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "status": { + "enum": [ + "declared", + "structural", + "protocol_default", + "inferred", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "confidence" + ], + "title": "EffectSemanticEvidence", + "type": "object" + }, + "EffectivePolicy": { + "additionalProperties": false, + "description": "Normalized effective-policy snapshot block (roadmap \u00a75.3).\n\nA semantic \u2014 not text \u2014 view of the release policy surface, so the\nbase-vs-head comparator can answer \"was the gate weakened?\" without a\ntext diff. Every field is derived deterministically from the manifest\n(plus, for ``baseline_fingerprints``, the accepted-debt findings), and\nevery list/dict is emitted in sorted order so two builds of the same\nmanifest serialize byte-identically.\n\nFields are chosen so the weakening / expansion classifiers can answer\nthe \u00a75.3 questions by set/rank comparison rather than counting:\n\n- ``ci_mode`` \u2014 was the CI gate weakened (strict -> advisory)?\n- ``fail_on`` \u2014 was the fail-on severity set loosened (head \u2282 base)?\n- ``severity_overrides`` \u2014 was a check's severity lowered across a tier?\n- ``suppressed_check_ids`` \u2014 was a blocking check newly suppressed?\n- ``waiver_scopes`` \u2014 was a waiver scope expanded (head \u228b base)?\n- ``baseline_fingerprints`` \u2014 was the accepted-debt baseline expanded?\n- ``baseline_integrity_mode`` \u2014 was baseline tamper-checking weakened?\n- ``ci_gate_present`` \u2014 was Shipgate CI removed from an opted-in repo?", + "properties": { + "baseline_fingerprints": { + "items": { + "type": "string" + }, + "title": "Baseline Fingerprints", + "type": "array" + }, + "baseline_integrity_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Baseline Integrity Mode" + }, + "ci_gate_present": { + "default": true, + "title": "Ci Gate Present", + "type": "boolean" + }, + "ci_mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ci Mode" + }, + "fail_on": { + "items": { + "type": "string" + }, + "title": "Fail On", + "type": "array" + }, + "severity_overrides": { + "additionalProperties": { + "type": "string" + }, + "title": "Severity Overrides", + "type": "object" + }, + "suppressed_check_ids": { + "items": { + "type": "string" + }, + "title": "Suppressed Check Ids", + "type": "array" + }, + "waiver_scopes": { + "items": { + "type": "string" + }, + "title": "Waiver Scopes", + "type": "array" + } + }, + "required": [ + "baseline_fingerprints", + "baseline_integrity_mode", + "ci_gate_present", + "ci_mode", + "fail_on", + "severity_overrides", + "suppressed_check_ids", + "waiver_scopes" + ], + "title": "EffectivePolicy", + "type": "object" + }, + "EvidenceCoverageDecision": { + "properties": { + "binding_coverage": { + "$ref": "#/$defs/BindingCoverageDecision" + }, + "evidence_gaps": { + "items": { + "$ref": "#/$defs/EvidenceGap" + }, + "title": "Evidence Gaps", + "type": "array" + }, + "human_review_recommended": { + "title": "Human Review Recommended", + "type": "boolean" + }, + "identity_coverage": { + "$ref": "#/$defs/IdentityCoverageDecision" + }, + "level": { + "title": "Level", + "type": "string" + }, + "low_confidence_tool_count": { + "title": "Low Confidence Tool Count", + "type": "integer" + }, + "semantic_coverage": { + "$ref": "#/$defs/SemanticCoverageDecision" + }, + "source_warning_count": { + "title": "Source Warning Count", + "type": "integer" + } + }, + "required": [ + "human_review_recommended", + "level", + "low_confidence_tool_count", + "source_warning_count" + ], + "title": "EvidenceCoverageDecision", + "type": "object" + }, + "EvidenceGap": { + "description": "v0.26: one structured row per measurable evidence gap.\n\n``insufficient_evidence`` previously diagnosed without prescribing;\neach gap names the degraded subject and the specific next action\nthat raises extraction confidence. Purely explanatory \u2014 gating\nstill uses only the counts (the gap list is a projection of them).", + "properties": { + "kind": { + "enum": [ + "low_confidence_tool", + "source_warning", + "incomplete_surface", + "missing_effect_evidence", + "inferred_effect_only", + "conflicting_effect_evidence", + "missing_authority_evidence", + "partial_authority_evidence", + "conflicting_authority_evidence", + "invalid_semantic_annotation", + "incomplete_tool_identity", + "conflicting_tool_identity", + "unresolved_tool_selector", + "ambiguous_tool_selector", + "ambiguous_legacy_tool_identity", + "invalid_tool_binding", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation" + ], + "title": "Kind", + "type": "string" + }, + "next_action": { + "$ref": "#/$defs/EvidenceGapAction" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Type" + }, + "subject": { + "title": "Subject", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "kind", + "subject", + "why", + "next_action" + ], + "title": "EvidenceGap", + "type": "object" + }, + "EvidenceGapAction": { + "description": "One concrete, mechanically-executable step that closes a gap.\n\nMirrors the agent-mode ``next_actions[]`` error shape\n(``kind``/``command``/``path``/``why``/``expects``) so agents reuse\none routing vocabulary across error recovery and evidence repair.", + "properties": { + "accepted_values": { + "items": { + "type": "string" + }, + "title": "Accepted Values", + "type": "array" + }, + "auto_apply": { + "const": false, + "default": false, + "title": "Auto Apply", + "type": "boolean" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Command" + }, + "declaration_template": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Declaration Template" + }, + "expects": { + "title": "Expects", + "type": "string" + }, + "kind": { + "enum": [ + "declare_tool_inventory", + "provide_source", + "review_warning", + "declare_action_effect", + "declare_action_authority", + "provide_complete_inventory", + "resolve_semantic_conflict", + "declare_source_identity", + "qualify_tool_selector", + "provide_tool_binding", + "resolve_tool_identity_conflict", + "regenerate_identity_artifact", + "declare_agent_root", + "declare_agent_bindings", + "provide_static_binding_source", + "provide_complete_binding_graph", + "resolve_binding_conflict", + "regenerate_binding_artifact" + ], + "title": "Kind", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "requires_human_review": { + "const": true, + "default": true, + "title": "Requires Human Review", + "type": "boolean" + }, + "suggested_patch_kind": { + "const": "manual", + "default": "manual", + "title": "Suggested Patch Kind", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "kind", + "why", + "expects" + ], + "title": "EvidenceGapAction", + "type": "object" + }, + "FailPolicy": { + "properties": { + "ci_mode": { + "title": "Ci Mode", + "type": "string" + }, + "exit_code": { + "title": "Exit Code", + "type": "integer" + }, + "fail_on": { + "items": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "type": "string" + }, + "title": "Fail On", + "type": "array" + }, + "new_findings_only": { + "default": false, + "title": "New Findings Only", + "type": "boolean" + }, + "would_fail_ci": { + "title": "Would Fail Ci", + "type": "boolean" + } + }, + "required": [ + "ci_mode", + "exit_code", + "fail_on", + "new_findings_only", + "would_fail_ci" + ], + "title": "FailPolicy", + "type": "object" + }, + "Finding": { + "additionalProperties": true, + "properties": { + "agent_action": { + "enum": [ + "auto_apply", + "propose_patch_for_review", + "escalate_to_human", + "suppress_with_reason", + "informational" + ], + "type": "string" + }, + "agent_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Agent Id" + }, + "autofix_safe": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Autofix Safe" + }, + "baseline_status": { + "anyOf": [ + { + "enum": [ + "new", + "matched", + "resolved" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Baseline Status" + }, + "blocks_release": { + "default": false, + "title": "Blocks Release", + "type": "boolean" + }, + "capability_policy_evidence": { + "anyOf": [ + { + "$ref": "#/$defs/CapabilityPolicyEvidence" + }, + { + "type": "null" + } + ], + "default": null + }, + "capability_refs": { + "items": { + "type": "string" + }, + "title": "Capability Refs", + "type": "array" + }, + "capability_trace_refs": { + "items": { + "type": "string" + }, + "title": "Capability Trace Refs", + "type": "array" + }, + "category": { + "title": "Category", + "type": "string" + }, + "check_id": { + "title": "Check Id", + "type": "string" + }, + "confidence": { + "default": "medium", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "docs_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Docs Url" + }, + "evidence": { + "additionalProperties": true, + "title": "Evidence", + "type": "object" + }, + "fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fingerprint" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "patches": { + "anyOf": [ + { + "items": { + "discriminator": { + "mapping": { + "append_pointer": "#/$defs/AppendPointerPatch", + "manual": "#/$defs/ManualPatch", + "remove_pointer": "#/$defs/RemovePointerPatch", + "set_pointer": "#/$defs/SetPointerPatch" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/SetPointerPatch" + }, + { + "$ref": "#/$defs/AppendPointerPatch" + }, + { + "$ref": "#/$defs/RemovePointerPatch" + }, + { + "$ref": "#/$defs/ManualPatch" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Patches" + }, + "policy_evidence_source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "policy_routing": { + "anyOf": [ + { + "$ref": "#/$defs/PolicyRoutingMetadata" + }, + { + "type": "null" + } + ], + "default": null + }, + "provenance_kind": { + "enum": [ + "static_declaration", + "ast_extraction", + "keyword_heuristic", + "regex_heuristic", + "policy_pack", + "runtime_trace" + ], + "type": "string" + }, + "recommendation": { + "title": "Recommendation", + "type": "string" + }, + "requires_human_review": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Requires Human Review" + }, + "severity": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Severity", + "type": "string" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "suggested_patch_kind": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Suggested Patch Kind" + }, + "suppressed": { + "default": false, + "title": "Suppressed", + "type": "boolean" + }, + "suppression_reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Suppression Reason" + }, + "title": { + "title": "Title", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + }, + "tool_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Name" + } + }, + "required": [ + "agent_action", + "baseline_status", + "blocks_release", + "capability_refs", + "category", + "check_id", + "confidence", + "evidence", + "fingerprint", + "id", + "provenance_kind", + "recommendation", + "severity", + "suppressed", + "title" + ], + "title": "Finding", + "type": "object" + }, + "HeuristicsFilter": { + "additionalProperties": false, + "description": "v0.21: top-level envelope describing the ``--no-heuristics``\nfilter pass.\n\nEmitted on every report regardless of whether the flag was set, so\nconsumers always read the same shape. When the flag is unset,\n``enabled=False`` and the count fields are zero \u2014 the active\nfinding set is unchanged. When the flag is set, every finding whose\n``provenance_kind`` is in ``excluded_provenance_kinds`` is marked\n``suppressed=True`` with ``suppression_reason=\"filtered by\n--no-heuristics\"`` BEFORE the release decision is built, so heuristic\nfindings can no longer gate release. Filtered findings stay in\n``findings[]`` for transparency; the audit envelope here records\naggregate counts.\n\nEarns the contract weight of ``Finding.provenance_kind`` (shipped\nv0.15) by giving it a first-class consumer. Same shape pattern as\n``PrivacyAudit``: an envelope that proves the filter ran and tells\na reviewer/agent which findings were excluded and why.", + "properties": { + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "excluded_provenance_kinds": { + "items": { + "type": "string" + }, + "title": "Excluded Provenance Kinds", + "type": "array" + }, + "filtered_by_kind": { + "additionalProperties": { + "type": "integer" + }, + "title": "Filtered By Kind", + "type": "object" + }, + "filtered_finding_count": { + "default": 0, + "title": "Filtered Finding Count", + "type": "integer" + } + }, + "required": [ + "enabled", + "excluded_provenance_kinds", + "filtered_by_kind", + "filtered_finding_count" + ], + "title": "HeuristicsFilter", + "type": "object" + }, + "HumanAck": { + "additionalProperties": false, + "description": "Human-acknowledgement state block (Decision 4 shape).\n\n``required`` is set by the human-ack builder when a trust-root\nweakening finding needs declared human authority; ``satisfied`` is\n``True`` when every required acknowledgement is present. ``acks``\nlists the declared entries; ``outstanding`` lists the surfaces still\nneeding one. The default empty shape is not required and therefore\nsatisfied.", + "properties": { + "acks": { + "items": { + "$ref": "#/$defs/HumanAckEntry" + }, + "title": "Acks", + "type": "array" + }, + "outstanding": { + "items": { + "type": "string" + }, + "title": "Outstanding", + "type": "array" + }, + "required": { + "default": false, + "title": "Required", + "type": "boolean" + }, + "satisfied": { + "default": true, + "title": "Satisfied", + "type": "boolean" + } + }, + "required": [ + "acks", + "outstanding", + "required", + "satisfied" + ], + "title": "HumanAck", + "type": "object" + }, + "HumanAckEntry": { + "additionalProperties": false, + "description": "One declared human-acknowledgement record.\n\nWithin the static boundary acknowledgement can only be *declared*\nevidence (roadmap \u00a75.4) \u2014 never inferred. An entry names the human\n``owner``, the ``reason``, an optional ``expires`` date (copied\nverbatim from the source, never generated), and the\n``affected_surface`` the ack covers.", + "properties": { + "affected_surface": { + "title": "Affected Surface", + "type": "string" + }, + "expires": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expires" + }, + "owner": { + "title": "Owner", + "type": "string" + }, + "reason": { + "title": "Reason", + "type": "string" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + } + }, + "required": [ + "affected_surface", + "expires", + "owner", + "reason", + "source" + ], + "title": "HumanAckEntry", + "type": "object" + }, + "IdentityCoverageDecision": { + "properties": { + "ambiguous_name_count": { + "default": 0, + "title": "Ambiguous Name Count", + "type": "integer" + }, + "bound_tools": { + "default": 0, + "title": "Bound Tools", + "type": "integer" + }, + "canonical_tools": { + "default": 0, + "title": "Canonical Tools", + "type": "integer" + }, + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible_tools": { + "default": 0, + "title": "Pass Eligible Tools", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "total_observations": { + "default": 0, + "title": "Total Observations", + "type": "integer" + } + }, + "title": "IdentityCoverageDecision", + "type": "object" + }, + "LoadedPolicyPack": { + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "owner": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Owner" + }, + "path": { + "title": "Path", + "type": "string" + }, + "rule_count": { + "title": "Rule Count", + "type": "integer" + }, + "sha256": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sha256" + }, + "sha256_status": { + "default": "unpinned", + "enum": [ + "unpinned", + "verified" + ], + "title": "Sha256 Status", + "type": "string" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Version" + } + }, + "required": [ + "id", + "name", + "path", + "rule_count" + ], + "title": "LoadedPolicyPack", + "type": "object" + }, + "ManualPatch": { + "additionalProperties": false, + "description": "No machine-applicable change. Carries human-readable instructions.\n\nUsed for every finding whose check ID has no v0.6 non-manual generator\nand for findings (like trace flips, per C6) that are intentionally\nnever auto-patched.", + "properties": { + "instructions": { + "title": "Instructions", + "type": "string" + }, + "kind": { + "const": "manual", + "default": "manual", + "title": "Kind", + "type": "string" + } + }, + "required": [ + "instructions" + ], + "title": "ManualPatch", + "type": "object" + }, + "Misalignment": { + "properties": { + "capability_refs": { + "items": { + "type": "string" + }, + "title": "Capability Refs", + "type": "array" + }, + "finding_refs": { + "items": { + "type": "string" + }, + "title": "Finding Refs", + "type": "array" + }, + "gap": { + "title": "Gap", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "intention_refs": { + "items": { + "type": "string" + }, + "title": "Intention Refs", + "type": "array" + }, + "kind": { + "enum": [ + "policy_gap", + "scope_drift", + "prohibited_action_present", + "control_missing", + "intent_mismatch", + "undetected_gap" + ], + "title": "Kind", + "type": "string" + }, + "policy_requirement": { + "title": "Policy Requirement", + "type": "string" + }, + "release_implication": { + "title": "Release Implication", + "type": "string" + }, + "severity": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Severity", + "type": "string" + }, + "tool_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Name" + } + }, + "required": [ + "capability_refs", + "finding_refs", + "gap", + "id", + "intention_refs", + "kind", + "policy_requirement", + "release_implication", + "severity", + "tool_name" + ], + "title": "Misalignment", + "type": "object" + }, + "PolicyApprovalRouting": { + "additionalProperties": false, + "description": "Non-enforcing approval-routing metadata from a policy-pack rule.", + "properties": { + "enforced": { + "const": false, + "default": false, + "title": "Enforced", + "type": "boolean" + }, + "min_approvals": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Min Approvals" + }, + "required": { + "default": false, + "title": "Required", + "type": "boolean" + }, + "teams": { + "items": { + "type": "string" + }, + "title": "Teams", + "type": "array" + } + }, + "title": "PolicyApprovalRouting", + "type": "object" + }, + "PolicyAudit": { + "additionalProperties": false, + "description": "v0.17 (M1) top-of-report audit envelope for policy decisions\napplied during scan.\n\nCarries severity-override audit today; M2 (baseline integrity) and\nM5 (plugin validation) will land sibling fields here so the audit\nenvelope stays stable across the trust-hardening releases.", + "properties": { + "severity_overrides_applied": { + "items": { + "$ref": "#/$defs/SeverityOverrideAuditEntry" + }, + "title": "Severity Overrides Applied", + "type": "array" + } + }, + "title": "PolicyAudit", + "type": "object" + }, + "PolicyRoutingMetadata": { + "additionalProperties": false, + "description": "Reviewer routing metadata carried outside finding evidence.", + "properties": { + "approval": { + "$ref": "#/$defs/PolicyApprovalRouting" + }, + "owner": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Owner" + }, + "reviewers": { + "items": { + "type": "string" + }, + "title": "Reviewers", + "type": "array" + } + }, + "title": "PolicyRoutingMetadata", + "type": "object" + }, + "PrivacyAudit": { + "additionalProperties": false, + "description": "Top-level audit envelope proving the default redaction pass ran.", + "properties": { + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "output_surfaces": { + "items": { + "type": "string" + }, + "title": "Output Surfaces", + "type": "array" + }, + "redacted_occurrence_count": { + "default": 0, + "title": "Redacted Occurrence Count", + "type": "integer" + }, + "redacted_paths": { + "items": { + "$ref": "#/$defs/RedactedPathSummary" + }, + "title": "Redacted Paths", + "type": "array" + }, + "rules_version": { + "title": "Rules Version", + "type": "string" + }, + "sensitive_field_inventory_version": { + "title": "Sensitive Field Inventory Version", + "type": "string" + } + }, + "required": [ + "enabled", + "notes", + "output_surfaces", + "redacted_occurrence_count", + "redacted_paths", + "rules_version", + "sensitive_field_inventory_version" + ], + "title": "PrivacyAudit", + "type": "object" + }, + "ProtectedSurfaceChange": { + "additionalProperties": false, + "description": "One touched protected path / policy surface (trust root).\n\nTier A trust-root protection (roadmap \u00a75.1/\u00a75.2) records *which*\nprotected surface a PR touched. ``path`` is the changed file; ``kind``\nis the classifier bucket it matched (e.g. ``manifest``, ``ci_gate``,\n``policy``); ``glob`` is the matched pattern. ``related_finding_ids``\nlinks to the ordinary ``SHIP-VERIFY-*`` findings that gate.", + "properties": { + "glob": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Glob" + }, + "kind": { + "default": "unknown", + "title": "Kind", + "type": "string" + }, + "path": { + "title": "Path", + "type": "string" + }, + "related_finding_ids": { + "items": { + "type": "string" + }, + "title": "Related Finding Ids", + "type": "array" + } + }, + "required": [ + "glob", + "kind", + "path", + "related_finding_ids" + ], + "title": "ProtectedSurfaceChange", + "type": "object" + }, + "RedactedPathSummary": { + "additionalProperties": false, + "description": "One privacy-audit row for a structural output path.\n\nThe row intentionally carries only aggregate counts and secret kinds,\nnever the original value or a hash/verifier of that value.", + "properties": { + "count": { + "default": 0, + "title": "Count", + "type": "integer" + }, + "kinds": { + "items": { + "type": "string" + }, + "title": "Kinds", + "type": "array" + }, + "path": { + "title": "Path", + "type": "string" + } + }, + "required": [ + "count", + "kinds", + "path" + ], + "title": "RedactedPathSummary", + "type": "object" + }, + "ReleaseConsequence": { + "properties": { + "blocker_misalignment_count": { + "default": 0, + "title": "Blocker Misalignment Count", + "type": "integer" + }, + "decision": { + "enum": [ + "blocked", + "review_required", + "insufficient_evidence", + "passed" + ], + "title": "Decision", + "type": "string" + }, + "fail_policy": { + "$ref": "#/$defs/FailPolicy" + }, + "review_misalignment_count": { + "default": 0, + "title": "Review Misalignment Count", + "type": "integer" + }, + "summary": { + "title": "Summary", + "type": "string" + } + }, + "required": [ + "blocker_misalignment_count", + "decision", + "fail_policy", + "review_misalignment_count", + "summary" + ], + "title": "ReleaseConsequence", + "type": "object" + }, + "ReleaseDecision": { + "properties": { + "baseline_delta": { + "$ref": "#/$defs/BaselineDelta" + }, + "blockers": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Blockers", + "type": "array" + }, + "contribution_rules": { + "items": { + "$ref": "#/$defs/ContributionRule" + }, + "title": "Contribution Rules", + "type": "array" + }, + "decision": { + "enum": [ + "blocked", + "review_required", + "insufficient_evidence", + "passed" + ], + "title": "Decision", + "type": "string" + }, + "evidence_coverage": { + "$ref": "#/$defs/EvidenceCoverageDecision" + }, + "fail_policy": { + "$ref": "#/$defs/FailPolicy" + }, + "reason": { + "title": "Reason", + "type": "string" + }, + "review_items": { + "items": { + "$ref": "#/$defs/ReleaseDecisionItem" + }, + "title": "Review Items", + "type": "array" + }, + "runtime_behavior_verified": { + "const": false, + "default": false, + "title": "Runtime Behavior Verified", + "type": "boolean" + }, + "static_analysis_only": { + "const": true, + "default": true, + "title": "Static Analysis Only", + "type": "boolean" + }, + "static_verdict_disclaimer": { + "default": "This verdict covers deterministic static evidence only. Agents Shipgate did not execute the agent or prove runtime behavior, tool routing, credential enforcement, or safety.", + "title": "Static Verdict Disclaimer", + "type": "string" + } + }, + "required": [ + "baseline_delta", + "blockers", + "contribution_rules", + "decision", + "evidence_coverage", + "fail_policy", + "reason", + "review_items" + ], + "title": "ReleaseDecision", + "type": "object" + }, + "ReleaseDecisionItem": { + "properties": { + "baseline_status": { + "anyOf": [ + { + "enum": [ + "new", + "matched", + "resolved" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Baseline Status" + }, + "blocks_release": { + "default": false, + "title": "Blocks Release", + "type": "boolean" + }, + "capability_refs": { + "items": { + "type": "string" + }, + "title": "Capability Refs", + "type": "array" + }, + "capability_trace_refs": { + "items": { + "type": "string" + }, + "title": "Capability Trace Refs", + "type": "array" + }, + "check_id": { + "title": "Check Id", + "type": "string" + }, + "fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fingerprint" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "policy_evidence_source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "severity": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Severity", + "type": "string" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceReference" + }, + { + "type": "null" + } + ], + "default": null + }, + "title": { + "title": "Title", + "type": "string" + } + }, + "required": [ + "baseline_status", + "blocks_release", + "capability_refs", + "check_id", + "fingerprint", + "id", + "severity", + "title" + ], + "title": "ReleaseDecisionItem", + "type": "object" + }, + "RemovePointerPatch": { + "additionalProperties": false, + "description": "Remove the node at a JSON pointer.", + "properties": { + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "kind": { + "const": "remove_pointer", + "default": "remove_pointer", + "title": "Kind", + "type": "string" + }, + "pointer": { + "title": "Pointer", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "target_file": { + "title": "Target File", + "type": "string" + }, + "target_format": { + "enum": [ + "yaml", + "json" + ], + "title": "Target Format", + "type": "string" + }, + "target_sha256": { + "title": "Target Sha256", + "type": "string" + } + }, + "required": [ + "target_file", + "pointer", + "target_format", + "confidence", + "rationale", + "target_sha256" + ], + "title": "RemovePointerPatch", + "type": "object" + }, + "ReportSummary": { + "properties": { + "critical_count": { + "default": 0, + "title": "Critical Count", + "type": "integer" + }, + "evidence_coverage": { + "default": "static", + "title": "Evidence Coverage", + "type": "string" + }, + "high_count": { + "default": 0, + "title": "High Count", + "type": "integer" + }, + "human_review_recommended": { + "default": false, + "title": "Human Review Recommended", + "type": "boolean" + }, + "info_count": { + "default": 0, + "title": "Info Count", + "type": "integer" + }, + "low_count": { + "default": 0, + "title": "Low Count", + "type": "integer" + }, + "medium_count": { + "default": 0, + "title": "Medium Count", + "type": "integer" + }, + "status": { + "title": "Status", + "type": "string" + }, + "suppressed_count": { + "default": 0, + "title": "Suppressed Count", + "type": "integer" + } + }, + "required": [ + "status" + ], + "title": "ReportSummary", + "type": "object" + }, + "ReviewerSummary": { + "additionalProperties": false, + "description": "Top-level summary block shaped for one-fetch reviewer consumption.\n\nDeterministic projection of the reviewer lens surfaces\n(``tool_surface_diff``, capability/intent diff, ``action_surface_diff``,\nevidence matrix) and the audit envelopes (``policy_audit``,\n``privacy_audit``, baseline integrity findings). A reviewer who wants\nheadline activity counts and a recommended starting surface can read\nthis block instead of opening every lens / audit envelope.\n\nParallels ``AgentSummary`` but for the audit/lens dimensions:\n``AgentSummary`` answers \"what should an agent do next?\" and\n``ReviewerSummary`` answers \"what should a reviewer look at first?\".\n\nAll fields are derived; this block cannot disagree with the\nunderlying lens/audit data.", + "properties": { + "action_surface_changes": { + "default": 0, + "title": "Action Surface Changes", + "type": "integer" + }, + "baseline_integrity_issues": { + "default": 0, + "title": "Baseline Integrity Issues", + "type": "integer" + }, + "capability_misalignments": { + "default": 0, + "title": "Capability Misalignments", + "type": "integer" + }, + "evidence_matrix_gaps": { + "default": 0, + "title": "Evidence Matrix Gaps", + "type": "integer" + }, + "first_recommended_surface": { + "anyOf": [ + { + "$ref": "#/$defs/ReviewerSurfacePointer" + }, + { + "type": "null" + } + ], + "default": null + }, + "headline": { + "title": "Headline", + "type": "string" + }, + "privacy_redactions": { + "default": 0, + "title": "Privacy Redactions", + "type": "integer" + }, + "severity_overrides_applied": { + "default": 0, + "title": "Severity Overrides Applied", + "type": "integer" + }, + "severity_overrides_tier_crossed": { + "default": 0, + "title": "Severity Overrides Tier Crossed", + "type": "integer" + }, + "tool_surface_changes": { + "default": 0, + "title": "Tool Surface Changes", + "type": "integer" + }, + "verdict": { + "enum": [ + "blocked", + "review_required", + "insufficient_evidence", + "passed" + ], + "title": "Verdict", + "type": "string" + } + }, + "required": [ + "action_surface_changes", + "baseline_integrity_issues", + "capability_misalignments", + "evidence_matrix_gaps", + "first_recommended_surface", + "headline", + "privacy_redactions", + "severity_overrides_applied", + "severity_overrides_tier_crossed", + "tool_surface_changes", + "verdict" + ], + "title": "ReviewerSummary", + "type": "object" + }, + "ReviewerSurfacePointer": { + "additionalProperties": false, + "description": "A single recommended reviewer entry point.\n\nMirrors the ``next_actions[]`` shape (kind/path/why) used by other\ncontract surfaces so the same renderer pattern works here. ``kind``\nclassifies the surface family (``release_decision`` /``lens`` /\n``audit`` /``evidence_matrix``); ``name`` is the canonical\nmachine-readable identifier; ``path`` is a dotted JSON path the\nreviewer can use to navigate. ``why`` is one sentence of reviewer\nrationale, suitable for a PR comment lead.", + "properties": { + "kind": { + "enum": [ + "release_decision", + "lens", + "audit", + "evidence_matrix" + ], + "title": "Kind", + "type": "string" + }, + "name": { + "enum": [ + "tool_surface_diff", + "capability_intent_diff", + "action_surface_diff", + "evidence_matrix", + "policy_audit", + "privacy_audit", + "baseline_integrity", + "release_decision" + ], + "title": "Name", + "type": "string" + }, + "path": { + "title": "Path", + "type": "string" + }, + "why": { + "title": "Why", + "type": "string" + } + }, + "required": [ + "kind", + "name", + "path", + "why" + ], + "title": "ReviewerSurfacePointer", + "type": "object" + }, + "SemanticClaimEvidence": { + "additionalProperties": false, + "properties": { + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "dimension": { + "enum": [ + "identity", + "binding", + "effect", + "authority" + ], + "title": "Dimension", + "type": "string" + }, + "evidence": { + "additionalProperties": true, + "title": "Evidence", + "type": "object" + }, + "provenance_kind": { + "enum": [ + "static_declaration", + "ast_extraction", + "keyword_heuristic", + "regex_heuristic", + "policy_pack", + "runtime_trace" + ], + "title": "Provenance Kind", + "type": "string" + }, + "source": { + "title": "Source", + "type": "string" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "dimension", + "value", + "confidence", + "provenance_kind", + "source" + ], + "title": "SemanticClaimEvidence", + "type": "object" + }, + "SemanticCoverageDecision": { + "description": "v0.29 pass eligibility across the normalized action surface.\n\nUnlike extraction-confidence thresholds, semantic gaps are\nzero-tolerance: any non-pass-eligible unknown/partial/conflicting\ndimension prevents ``passed``. Known authority review concerns (for\nexample ambient or unscoped credentials) are counted separately so\nthey deterministically route to ``review_required`` rather than\n``insufficient_evidence``.", + "properties": { + "gap_count": { + "default": 0, + "title": "Gap Count", + "type": "integer" + }, + "pass_eligible_actions": { + "default": 0, + "title": "Pass Eligible Actions", + "type": "integer" + }, + "reason_counts": { + "additionalProperties": { + "type": "integer" + }, + "title": "Reason Counts", + "type": "object" + }, + "review_concern_count": { + "default": 0, + "title": "Review Concern Count", + "type": "integer" + }, + "total_actions": { + "default": 0, + "title": "Total Actions", + "type": "integer" + } + }, + "title": "SemanticCoverageDecision", + "type": "object" + }, + "SemanticIssueEvidence": { + "additionalProperties": false, + "properties": { + "dimension": { + "enum": [ + "identity", + "binding", + "effect", + "authority" + ], + "title": "Dimension", + "type": "string" + }, + "kind": { + "enum": [ + "incomplete_tool_identity", + "conflicting_tool_identity", + "unresolved_tool_selector", + "ambiguous_tool_selector", + "ambiguous_legacy_tool_identity", + "invalid_tool_binding", + "incomplete_surface", + "missing_effect_evidence", + "inferred_effect_only", + "conflicting_effect_evidence", + "missing_authority_evidence", + "partial_authority_evidence", + "conflicting_authority_evidence", + "invalid_semantic_annotation", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation" + ], + "title": "Kind", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + }, + "source_pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Pointer" + } + }, + "required": [ + "kind", + "dimension", + "message" + ], + "title": "SemanticIssueEvidence", + "type": "object" + }, + "SetPointerPatch": { + "additionalProperties": false, + "description": "Set the value at a JSON pointer inside a YAML or JSON file.", + "properties": { + "confidence": { + "enum": [ + "low", + "medium", + "high" + ], + "title": "Confidence", + "type": "string" + }, + "kind": { + "const": "set_pointer", + "default": "set_pointer", + "title": "Kind", + "type": "string" + }, + "pointer": { + "title": "Pointer", + "type": "string" + }, + "rationale": { + "title": "Rationale", + "type": "string" + }, + "target_file": { + "title": "Target File", + "type": "string" + }, + "target_format": { + "enum": [ + "yaml", + "json" + ], + "title": "Target Format", + "type": "string" + }, + "target_sha256": { + "title": "Target Sha256", + "type": "string" + }, + "value": { + "title": "Value" + } + }, + "required": [ + "target_file", + "pointer", + "value", + "target_format", + "confidence", + "rationale", + "target_sha256" + ], + "title": "SetPointerPatch", + "type": "object" + }, + "SeverityOverrideAuditEntry": { + "additionalProperties": false, + "description": "One row in ``ReadinessReport.policy_audit.severity_overrides_applied``.\n\nv0.17 (M1). Surfaces every manifest-driven severity override so a\nreviewer can see what was downgraded (or upgraded) without diving\ninto per-finding evidence. Emitted regardless of whether the override\nmatched any active finding \u2014 entries for checks that did not fire\nstill document reviewer intent.", + "properties": { + "applied_severity": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Applied Severity", + "type": "string" + }, + "check_id": { + "title": "Check Id", + "type": "string" + }, + "default_severity": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Default Severity", + "type": "string" + }, + "direction": { + "default": "same", + "enum": [ + "downgrade", + "upgrade", + "same" + ], + "title": "Direction", + "type": "string" + }, + "expires": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expires" + }, + "manifest_path": { + "title": "Manifest Path", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reason" + }, + "tier_crossed": { + "default": false, + "title": "Tier Crossed", + "type": "boolean" + } + }, + "required": [ + "check_id", + "default_severity", + "applied_severity", + "manifest_path" + ], + "title": "SeverityOverrideAuditEntry", + "type": "object" + }, + "SourceReference": { + "additionalProperties": true, + "properties": { + "end_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "End Line" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "pointer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pointer" + }, + "ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ref" + }, + "start_column": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Start Column" + }, + "start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Start Line" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "SourceReference", + "type": "object" + }, + "SuggestedScenario": { + "properties": { + "expected_control": { + "title": "Expected Control", + "type": "string" + }, + "given": { + "title": "Given", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "scenario_type": { + "enum": [ + "approval", + "confirmation", + "idempotency_retry", + "least_privilege_scope", + "prohibited_action", + "wildcard_inventory", + "schema_boundary", + "prompt_scope_alignment", + "test_case_coverage" + ], + "title": "Scenario Type", + "type": "string" + }, + "source_findings": { + "items": { + "type": "string" + }, + "title": "Source Findings", + "type": "array" + }, + "source_misalignments": { + "items": { + "type": "string" + }, + "title": "Source Misalignments", + "type": "array" + }, + "title": { + "title": "Title", + "type": "string" + } + }, + "required": [ + "expected_control", + "given", + "id", + "scenario_type", + "source_findings", + "source_misalignments", + "title" + ], + "title": "SuggestedScenario", + "type": "object" + }, + "ToolIdentityEvidence": { + "additionalProperties": false, + "properties": { + "binding_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Binding Id" + }, + "claims": { + "items": { + "$ref": "#/$defs/SemanticClaimEvidence" + }, + "title": "Claims", + "type": "array" + }, + "issues": { + "items": { + "$ref": "#/$defs/SemanticIssueEvidence" + }, + "title": "Issues", + "type": "array" + }, + "observation_ids": { + "items": { + "type": "string" + }, + "title": "Observation Ids", + "type": "array" + }, + "pass_eligible": { + "title": "Pass Eligible", + "type": "boolean" + }, + "primary_observation_id": { + "title": "Primary Observation Id", + "type": "string" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "status": { + "enum": [ + "declared", + "structural", + "partial", + "unknown", + "conflicting" + ], + "title": "Status", + "type": "string" + }, + "tool_id": { + "title": "Tool Id", + "type": "string" + } + }, + "required": [ + "tool_id", + "status", + "provider", + "primary_observation_id", + "pass_eligible" + ], + "title": "ToolIdentityEvidence", + "type": "object" + }, + "ToolSemanticEvidence": { + "additionalProperties": false, + "description": "Public semantic assessment attached to action and capability facts.", + "properties": { + "authority": { + "$ref": "#/$defs/AuthoritySemanticEvidence" + }, + "binding": { + "$ref": "#/$defs/BindingSemanticEvidence" + }, + "conservative_effect": { + "enum": [ + "read", + "write", + "destructive", + "external_communication", + "financial_write", + "production_operation", + "privileged_data_access", + "code_execution", + "identity_access" + ], + "title": "Conservative Effect", + "type": "string" + }, + "effect": { + "$ref": "#/$defs/EffectSemanticEvidence" + }, + "identity": { + "anyOf": [ + { + "$ref": "#/$defs/ToolIdentityEvidence" + }, + { + "type": "null" + } + ], + "default": null + }, + "pass_eligible": { + "title": "Pass Eligible", + "type": "boolean" + } + }, + "required": [ + "conservative_effect", + "effect", + "authority", + "pass_eligible" + ], + "title": "ToolSemanticEvidence", + "type": "object" + }, + "ToolSurfaceControlChange": { + "additionalProperties": false, + "properties": { + "control": { + "enum": [ + "approval_policy", + "confirmation_policy", + "idempotency_evidence" + ], + "title": "Control", + "type": "string" + }, + "kind": { + "enum": [ + "added", + "removed", + "changed" + ], + "title": "Kind", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reason" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "tool": { + "title": "Tool", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "control", + "kind", + "reason", + "source", + "tool" + ], + "title": "ToolSurfaceControlChange", + "type": "object" + }, + "ToolSurfaceControlFact": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "approval_policy", + "confirmation_policy", + "idempotency_evidence" + ], + "title": "Kind", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reason" + }, + "source": { + "title": "Source", + "type": "string" + }, + "tool": { + "title": "Tool", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "kind", + "reason", + "source", + "tool" + ], + "title": "ToolSurfaceControlFact", + "type": "object" + }, + "ToolSurfaceDiff": { + "additionalProperties": false, + "properties": { + "base": { + "$ref": "#/$defs/ToolSurfaceDiffBase" + }, + "controls": { + "items": { + "$ref": "#/$defs/ToolSurfaceControlChange" + }, + "title": "Controls", + "type": "array" + }, + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "finding_deltas": { + "$ref": "#/$defs/ToolSurfaceFindingDeltas" + }, + "high_risk_effects": { + "items": { + "$ref": "#/$defs/ToolSurfaceHighRiskEffectChange" + }, + "title": "High Risk Effects", + "type": "array" + }, + "metadata_changes": { + "items": { + "$ref": "#/$defs/ToolSurfaceMetadataChange" + }, + "title": "Metadata Changes", + "type": "array" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + }, + "policy_drift": { + "items": { + "$ref": "#/$defs/ToolSurfacePolicyDrift" + }, + "title": "Policy Drift", + "type": "array" + }, + "scopes": { + "items": { + "$ref": "#/$defs/ToolSurfaceScopeChange" + }, + "title": "Scopes", + "type": "array" + }, + "summary": { + "$ref": "#/$defs/ToolSurfaceDiffSummary" + }, + "tools": { + "items": { + "$ref": "#/$defs/ToolSurfaceToolChange" + }, + "title": "Tools", + "type": "array" + } + }, + "required": [ + "base", + "controls", + "enabled", + "finding_deltas", + "high_risk_effects", + "metadata_changes", + "notes", + "policy_drift", + "scopes", + "summary", + "tools" + ], + "title": "ToolSurfaceDiff", + "type": "object" + }, + "ToolSurfaceDiffBase": { + "additionalProperties": false, + "properties": { + "baseline_schema_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Baseline Schema Version" + }, + "kind": { + "default": "none", + "enum": [ + "none", + "report", + "baseline" + ], + "title": "Kind", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "report_schema_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Report Schema Version" + } + }, + "required": [ + "baseline_schema_version", + "kind", + "path", + "report_schema_version" + ], + "title": "ToolSurfaceDiffBase", + "type": "object" + }, + "ToolSurfaceDiffSummary": { + "additionalProperties": false, + "properties": { + "accepted_debt": { + "default": 0, + "title": "Accepted Debt", + "type": "integer" + }, + "controls_added": { + "default": 0, + "title": "Controls Added", + "type": "integer" + }, + "controls_removed": { + "default": 0, + "title": "Controls Removed", + "type": "integer" + }, + "metadata_changes": { + "default": 0, + "title": "Metadata Changes", + "type": "integer" + }, + "new_findings": { + "default": 0, + "title": "New Findings", + "type": "integer" + }, + "new_high_risk_effects": { + "default": 0, + "title": "New High Risk Effects", + "type": "integer" + }, + "new_scopes": { + "default": 0, + "title": "New Scopes", + "type": "integer" + }, + "policy_drift_items": { + "default": 0, + "title": "Policy Drift Items", + "type": "integer" + }, + "removed_high_risk_effects": { + "default": 0, + "title": "Removed High Risk Effects", + "type": "integer" + }, + "removed_scopes": { + "default": 0, + "title": "Removed Scopes", + "type": "integer" + }, + "resolved_findings": { + "default": 0, + "title": "Resolved Findings", + "type": "integer" + }, + "tools_added": { + "default": 0, + "title": "Tools Added", + "type": "integer" + }, + "tools_changed": { + "default": 0, + "title": "Tools Changed", + "type": "integer" + }, + "tools_removed": { + "default": 0, + "title": "Tools Removed", + "type": "integer" + }, + "unchanged_findings": { + "default": 0, + "title": "Unchanged Findings", + "type": "integer" + } + }, + "required": [ + "accepted_debt", + "controls_added", + "controls_removed", + "metadata_changes", + "new_findings", + "new_high_risk_effects", + "new_scopes", + "policy_drift_items", + "removed_high_risk_effects", + "removed_scopes", + "resolved_findings", + "tools_added", + "tools_changed", + "tools_removed", + "unchanged_findings" + ], + "title": "ToolSurfaceDiffSummary", + "type": "object" + }, + "ToolSurfaceFacts": { + "additionalProperties": false, + "properties": { + "controls": { + "items": { + "$ref": "#/$defs/ToolSurfaceControlFact" + }, + "title": "Controls", + "type": "array" + }, + "policies": { + "items": { + "$ref": "#/$defs/ToolSurfacePolicyFact" + }, + "title": "Policies", + "type": "array" + }, + "scopes": { + "items": { + "$ref": "#/$defs/ToolSurfaceScopeFact" + }, + "title": "Scopes", + "type": "array" + }, + "tools": { + "items": { + "$ref": "#/$defs/ToolSurfaceToolFact" + }, + "title": "Tools", + "type": "array" + } + }, + "required": [ + "controls", + "policies", + "scopes", + "tools" + ], + "title": "ToolSurfaceFacts", + "type": "object" + }, + "ToolSurfaceFieldChange": { + "additionalProperties": false, + "properties": { + "after": { + "default": null, + "title": "After" + }, + "before": { + "default": null, + "title": "Before" + }, + "field": { + "title": "Field", + "type": "string" + } + }, + "required": [ + "after", + "before", + "field" + ], + "title": "ToolSurfaceFieldChange", + "type": "object" + }, + "ToolSurfaceFindingDeltaItem": { + "additionalProperties": false, + "properties": { + "baseline_status": { + "anyOf": [ + { + "enum": [ + "new", + "matched", + "resolved" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Baseline Status" + }, + "check_id": { + "title": "Check Id", + "type": "string" + }, + "fingerprint": { + "title": "Fingerprint", + "type": "string" + }, + "severity": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "title": "Severity", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "tool_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Name" + } + }, + "required": [ + "baseline_status", + "check_id", + "fingerprint", + "severity", + "title", + "tool_name" + ], + "title": "ToolSurfaceFindingDeltaItem", + "type": "object" + }, + "ToolSurfaceFindingDeltas": { + "additionalProperties": false, + "properties": { + "accepted_debt": { + "items": { + "$ref": "#/$defs/ToolSurfaceFindingDeltaItem" + }, + "title": "Accepted Debt", + "type": "array" + }, + "new_findings": { + "items": { + "$ref": "#/$defs/ToolSurfaceFindingDeltaItem" + }, + "title": "New Findings", + "type": "array" + }, + "resolved_findings": { + "items": { + "$ref": "#/$defs/ToolSurfaceFindingDeltaItem" + }, + "title": "Resolved Findings", + "type": "array" + }, + "unchanged_findings": { + "items": { + "$ref": "#/$defs/ToolSurfaceFindingDeltaItem" + }, + "title": "Unchanged Findings", + "type": "array" + } + }, + "required": [ + "accepted_debt", + "new_findings", + "resolved_findings", + "unchanged_findings" + ], + "title": "ToolSurfaceFindingDeltas", + "type": "object" + }, + "ToolSurfaceHashes": { + "additionalProperties": false, + "properties": { + "annotations": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Annotations" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "input_schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Schema" + }, + "output_schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Output Schema" + }, + "parameters": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parameters" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + } + }, + "required": [ + "annotations", + "description", + "input_schema", + "output_schema", + "parameters", + "source_ref" + ], + "title": "ToolSurfaceHashes", + "type": "object" + }, + "ToolSurfaceHighRiskEffectChange": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "added", + "removed", + "changed" + ], + "title": "Kind", + "type": "string" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "tag": { + "title": "Tag", + "type": "string" + }, + "tool": { + "title": "Tool", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "kind", + "tag", + "tool" + ], + "title": "ToolSurfaceHighRiskEffectChange", + "type": "object" + }, + "ToolSurfaceMetadataChange": { + "additionalProperties": false, + "properties": { + "after": { + "default": null, + "title": "After" + }, + "before": { + "default": null, + "title": "Before" + }, + "kind": { + "enum": [ + "added", + "removed", + "changed" + ], + "title": "Kind", + "type": "string" + }, + "metadata": { + "title": "Metadata", + "type": "string" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "tool": { + "title": "Tool", + "type": "string" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "after", + "before", + "kind", + "metadata", + "tool" + ], + "title": "ToolSurfaceMetadataChange", + "type": "object" + }, + "ToolSurfacePolicyDrift": { + "additionalProperties": false, + "properties": { + "after_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "After Hash" + }, + "after_summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "After Summary" + }, + "before_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Before Hash" + }, + "before_summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Before Summary" + }, + "key": { + "title": "Key", + "type": "string" + }, + "kind": { + "enum": [ + "added", + "removed", + "changed" + ], + "title": "Kind", + "type": "string" + }, + "policy_kind": { + "title": "Policy Kind", + "type": "string" + } + }, + "required": [ + "after_hash", + "after_summary", + "before_hash", + "before_summary", + "key", + "kind", + "policy_kind" + ], + "title": "ToolSurfacePolicyDrift", + "type": "object" + }, + "ToolSurfacePolicyFact": { + "additionalProperties": false, + "properties": { + "key": { + "title": "Key", + "type": "string" + }, + "kind": { + "title": "Kind", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "value_hash": { + "title": "Value Hash", + "type": "string" + } + }, + "required": [ + "key", + "kind", + "summary", + "value_hash" + ], + "title": "ToolSurfacePolicyFact", + "type": "object" + }, + "ToolSurfaceScopeChange": { + "additionalProperties": false, + "properties": { + "broad": { + "default": false, + "title": "Broad", + "type": "boolean" + }, + "kind": { + "enum": [ + "added", + "removed", + "changed" + ], + "title": "Kind", + "type": "string" + }, + "scope": { + "title": "Scope", + "type": "string" + }, + "scope_kind": { + "enum": [ + "tool_required", + "manifest_declared" + ], + "title": "Scope Kind", + "type": "string" + }, + "tool_ids": { + "items": { + "type": "string" + }, + "title": "Tool Ids", + "type": "array" + }, + "tool_names": { + "items": { + "type": "string" + }, + "title": "Tool Names", + "type": "array" + } + }, + "required": [ + "broad", + "kind", + "scope", + "scope_kind", + "tool_names" + ], + "title": "ToolSurfaceScopeChange", + "type": "object" + }, + "ToolSurfaceScopeFact": { + "additionalProperties": false, + "properties": { + "broad": { + "default": false, + "title": "Broad", + "type": "boolean" + }, + "kind": { + "enum": [ + "tool_required", + "manifest_declared" + ], + "title": "Kind", + "type": "string" + }, + "scope": { + "title": "Scope", + "type": "string" + }, + "tool_ids": { + "items": { + "type": "string" + }, + "title": "Tool Ids", + "type": "array" + }, + "tool_names": { + "items": { + "type": "string" + }, + "title": "Tool Names", + "type": "array" + } + }, + "required": [ + "broad", + "kind", + "scope", + "tool_names" + ], + "title": "ToolSurfaceScopeFact", + "type": "object" + }, + "ToolSurfaceSummary": { + "properties": { + "high_risk_tools": { + "title": "High Risk Tools", + "type": "integer" + }, + "missing_descriptions": { + "default": 0, + "title": "Missing Descriptions", + "type": "integer" + }, + "sources": { + "additionalProperties": { + "type": "integer" + }, + "title": "Sources", + "type": "object" + }, + "total_tools": { + "title": "Total Tools", + "type": "integer" + }, + "wildcard_tools": { + "default": 0, + "title": "Wildcard Tools", + "type": "integer" + } + }, + "required": [ + "total_tools", + "high_risk_tools" + ], + "title": "ToolSurfaceSummary", + "type": "object" + }, + "ToolSurfaceToolChange": { + "additionalProperties": false, + "properties": { + "changes": { + "items": { + "$ref": "#/$defs/ToolSurfaceFieldChange" + }, + "title": "Changes", + "type": "array" + }, + "kind": { + "enum": [ + "added", + "removed", + "changed" + ], + "title": "Kind", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "provider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Provider" + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Path" + }, + "source_start_line": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Start Line" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Type" + }, + "tool_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tool Id" + } + }, + "required": [ + "changes", + "kind", + "name", + "source_id", + "source_type" + ], + "title": "ToolSurfaceToolChange", + "type": "object" + }, + "ToolSurfaceToolFact": { + "additionalProperties": false, + "properties": { + "auth_scopes": { + "items": { + "type": "string" + }, + "title": "Auth Scopes", + "type": "array" + }, + "extraction_confidence": { + "default": "low", + "enum": [ + "low", + "medium", + "high" + ], + "title": "Extraction Confidence", + "type": "string" + }, + "has_description": { + "default": false, + "title": "Has Description", + "type": "boolean" + }, + "hashes": { + "$ref": "#/$defs/ToolSurfaceHashes" + }, + "name": { + "title": "Name", + "type": "string" + }, + "owner": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Owner" + }, + "provider": { + "default": "", + "title": "Provider", + "type": "string" + }, + "risk_tags": { + "items": { + "type": "string" + }, + "title": "Risk Tags", + "type": "array" + }, + "source_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Id" + }, + "source_ref": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Ref" + }, + "source_type": { + "title": "Source Type", + "type": "string" + }, + "tool_id": { + "default": "", + "title": "Tool Id", + "type": "string" + } + }, + "required": [ + "auth_scopes", + "extraction_confidence", + "has_description", + "hashes", + "name", + "owner", + "risk_tags", + "source_id", + "source_ref", + "source_type" + ], + "title": "ToolSurfaceToolFact", + "type": "object" + }, + "VerifierCapabilityDeltaSummary": { + "additionalProperties": false, + "description": "Counts of the capability delta, by direction.\n\nEqual by construction to the lengths of the matching\n``CapabilityChangeBlock`` member lists (roadmap \u00a78 required test:\nreviewer/verifier capability counts equal the underlying rollup).", + "properties": { + "added": { + "default": 0, + "title": "Added", + "type": "integer" + }, + "broadened": { + "default": 0, + "title": "Broadened", + "type": "integer" + }, + "narrowed": { + "default": 0, + "title": "Narrowed", + "type": "integer" + }, + "removed": { + "default": 0, + "title": "Removed", + "type": "integer" + } + }, + "required": [ + "added", + "broadened", + "narrowed", + "removed" + ], + "title": "VerifierCapabilityDeltaSummary", + "type": "object" + }, + "VerifierReasonCodeCount": { + "additionalProperties": false, + "description": "One ``(reason_code, count)`` pair for the verifier summary.\n\nA list of these (rather than a dict) keeps the ordering explicit and\nbyte-stable; the builder sorts them deterministically.", + "properties": { + "count": { + "default": 0, + "title": "Count", + "type": "integer" + }, + "reason_code": { + "title": "Reason Code", + "type": "string" + } + }, + "required": [ + "count", + "reason_code" + ], + "title": "VerifierReasonCodeCount", + "type": "object" + }, + "VerifierSummary": { + "additionalProperties": false, + "description": "Top-level verifier composition alias (Decision 5 shape).\n\nA byte-stable projection bundling the release verdict, finding counts,\nthe capability delta summary, and trust-root / acknowledgement flags\nfor one-fetch controller consumption. Composition only \u2014 it cannot\nintroduce a finding-independent blocker, and ``verdict`` MUST mirror\n``release_decision.decision`` (Principle 2; enforced by a property\ntest). Parallels ``agent_summary`` / ``reviewer_summary``.\n\n``top_reason_codes`` is capped at the top five by the builder, ranked\nseverity desc, count desc, then reason code asc.", + "properties": { + "by_reason_code": { + "additionalProperties": { + "type": "integer" + }, + "title": "By Reason Code", + "type": "object" + }, + "by_severity": { + "additionalProperties": { + "type": "integer" + }, + "title": "By Severity", + "type": "object" + }, + "capability_delta_summary": { + "$ref": "#/$defs/VerifierCapabilityDeltaSummary" + }, + "human_ack_required": { + "default": false, + "title": "Human Ack Required", + "type": "boolean" + }, + "human_ack_satisfied": { + "default": true, + "title": "Human Ack Satisfied", + "type": "boolean" + }, + "policy_weakened": { + "default": false, + "title": "Policy Weakened", + "type": "boolean" + }, + "protected_surface_touched": { + "default": false, + "title": "Protected Surface Touched", + "type": "boolean" + }, + "top_reason_codes": { + "items": { + "$ref": "#/$defs/VerifierReasonCodeCount" + }, + "title": "Top Reason Codes", + "type": "array" + }, + "verdict": { + "enum": [ + "blocked", + "review_required", + "insufficient_evidence", + "passed" + ], + "title": "Verdict", + "type": "string" + } + }, + "required": [ + "by_reason_code", + "by_severity", + "capability_delta_summary", + "human_ack_required", + "human_ack_satisfied", + "policy_weakened", + "protected_surface_touched", + "top_reason_codes", + "verdict" + ], + "title": "VerifierSummary", + "type": "object" + } + }, + "$id": "https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/report-schema.v0.31.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "description": "JSON Schema for the Agents Shipgate Tool-Use Readiness Report. Generated from agents_shipgate.schemas.report.ReadinessReport with post-processing to preserve the v0.5 public contract. Do not edit by hand.", + "properties": { + "action_surface_diff": { + "$ref": "#/$defs/ActionSurfaceDiff" + }, + "action_surface_facts": { + "$ref": "#/$defs/ActionSurfaceFacts" + }, + "agent": { + "additionalProperties": true, + "title": "Agent", + "type": "object" + }, + "agent_summary": { + "$ref": "#/$defs/AgentSummary" + }, + "anthropic_surface": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Anthropic Surface" + }, + "api_surface": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Surface" + }, + "baseline": { + "anyOf": [ + { + "$ref": "#/$defs/BaselineSummary" + }, + { + "type": "null" + } + ], + "default": null + }, + "binding_surface_diff": { + "$ref": "#/$defs/BindingSurfaceDiff" + }, + "binding_surface_facts": { + "$ref": "#/$defs/AgentBindingGraphAssessment" + }, + "capability_change": { + "$ref": "#/$defs/CapabilityChangeBlock" + }, + "capability_facts": { + "items": { + "$ref": "#/$defs/CapabilityFact" + }, + "title": "Capability Facts", + "type": "array" + }, + "capability_runtime_evidence": { + "$ref": "#/$defs/CapabilityRuntimeEvidence" + }, + "codex_plugin_surface": { + "anyOf": [ + { + "$ref": "#/$defs/CodexPluginSurface" + }, + { + "type": "null" + } + ], + "default": null + }, + "declared_intentions": { + "items": { + "$ref": "#/$defs/DeclaredIntention" + }, + "title": "Declared Intentions", + "type": "array" + }, + "effective_policy": { + "$ref": "#/$defs/EffectivePolicy" + }, + "environment": { + "additionalProperties": true, + "title": "Environment", + "type": "object" + }, + "findings": { + "items": { + "$ref": "#/$defs/Finding" + }, + "title": "Findings", + "type": "array" + }, + "frameworks": { + "additionalProperties": true, + "properties": { + "crewai": { + "additionalProperties": true, + "required": [ + "agent_count", + "class_tool_count", + "crew_count", + "dynamic_tool_surface_count", + "function_tool_count", + "prebuilt_tool_count", + "python_entrypoint_count", + "tool_inventory_file_count", + "warnings" + ], + "type": "object" + }, + "google_adk": { + "additionalProperties": true, + "required": [ + "agent_config_count", + "agent_count", + "callback_count", + "dynamic_toolset_count", + "eval_file_count", + "function_tool_count", + "long_running_tool_count", + "plugin_count", + "python_entrypoint_count", + "sub_agent_count", + "tool_inventory_file_count", + "toolset_count", + "trace_sample_count", + "warnings" + ], + "type": "object" + }, + "langchain": { + "additionalProperties": true, + "required": [ + "agent_tool_binding_count", + "dynamic_tool_surface_count", + "function_tool_count", + "python_entrypoint_count", + "structured_tool_count", + "tool_inventory_file_count", + "tool_node_count", + "warnings" + ], + "type": "object" + } + }, + "title": "Frameworks", + "type": "object" + }, + "generated_reports": { + "additionalProperties": { + "type": "string" + }, + "title": "Generated Reports", + "type": "object" + }, + "heuristics_filter": { + "$ref": "#/$defs/HeuristicsFilter" + }, + "human_ack": { + "$ref": "#/$defs/HumanAck" + }, + "loaded_adapters": { + "items": { + "additionalProperties": true, + "required": [ + "distribution", + "name", + "runtime_errors", + "source_type", + "validation_errors", + "validation_status", + "value", + "version" + ], + "type": "object" + }, + "title": "Loaded Adapters", + "type": "array" + }, + "loaded_plugins": { + "items": { + "additionalProperties": true, + "required": [ + "check_id", + "distribution", + "name", + "runtime_errors", + "validation_errors", + "validation_status", + "value", + "version" + ], + "type": "object" + }, + "title": "Loaded Plugins", + "type": "array" + }, + "loaded_policy_packs": { + "items": { + "$ref": "#/$defs/LoadedPolicyPack" + }, + "title": "Loaded Policy Packs", + "type": "array" + }, + "manifest_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Manifest Dir" + }, + "misalignments": { + "items": { + "$ref": "#/$defs/Misalignment" + }, + "title": "Misalignments", + "type": "array" + }, + "policy_audit": { + "$ref": "#/$defs/PolicyAudit" + }, + "privacy_audit": { + "$ref": "#/$defs/PrivacyAudit" + }, + "project": { + "additionalProperties": true, + "title": "Project", + "type": "object" + }, + "protected_surface_changes": { + "items": { + "$ref": "#/$defs/ProtectedSurfaceChange" + }, + "title": "Protected Surface Changes", + "type": "array" + }, + "recommended_actions": { + "items": { + "type": "string" + }, + "title": "Recommended Actions", + "type": "array" + }, + "release_consequence": { + "$ref": "#/$defs/ReleaseConsequence" + }, + "release_decision": { + "$ref": "#/$defs/ReleaseDecision" + }, + "report_schema_version": { + "const": "0.31" + }, + "reviewer_summary": { + "$ref": "#/$defs/ReviewerSummary" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "schema_version": { + "const": "0.1" + }, + "source_warnings": { + "items": { + "type": "string" + }, + "title": "Source Warnings", + "type": "array" + }, + "suggested_scenarios": { + "items": { + "$ref": "#/$defs/SuggestedScenario" + }, + "title": "Suggested Scenarios", + "type": "array" + }, + "summary": { + "$ref": "#/$defs/ReportSummary" + }, + "tool_catalog": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Tool Catalog", + "type": "array" + }, + "tool_inventory": { + "items": { + "additionalProperties": true, + "required": [ + "auth_scopes", + "confidence", + "name", + "risk_tags", + "source_type" + ], + "type": "object" + }, + "title": "Tool Inventory", + "type": "array" + }, + "tool_surface": { + "$ref": "#/$defs/ToolSurfaceSummary" + }, + "tool_surface_diff": { + "$ref": "#/$defs/ToolSurfaceDiff" + }, + "tool_surface_facts": { + "$ref": "#/$defs/ToolSurfaceFacts" + }, + "verifier_summary": { + "$ref": "#/$defs/VerifierSummary" + } + }, + "required": [ + "action_surface_diff", + "action_surface_facts", + "agent", + "agent_summary", + "capability_change", + "capability_facts", + "codex_plugin_surface", + "declared_intentions", + "effective_policy", + "environment", + "findings", + "frameworks", + "generated_reports", + "heuristics_filter", + "human_ack", + "loaded_adapters", + "loaded_plugins", + "loaded_policy_packs", + "misalignments", + "policy_audit", + "privacy_audit", + "project", + "protected_surface_changes", + "recommended_actions", + "release_consequence", + "release_decision", + "report_schema_version", + "reviewer_summary", + "run_id", + "schema_version", + "source_warnings", + "suggested_scenarios", + "summary", + "tool_inventory", + "tool_surface", + "tool_surface_diff", + "tool_surface_facts", + "verifier_summary" + ], + "title": "Agents Shipgate Readiness Report v0.31", + "type": "object" +} diff --git a/docs/report-sensitive-fields.json b/docs/report-sensitive-fields.json index 4a77a306..81319585 100644 --- a/docs/report-sensitive-fields.json +++ b/docs/report-sensitive-fields.json @@ -30,6 +30,8 @@ {"surface": "report", "path": "tool_surface_diff", "classification": "hash_only"}, {"surface": "report", "path": "action_surface_facts", "classification": "schema_metadata"}, {"surface": "report", "path": "action_surface_diff", "classification": "schema_metadata"}, + {"surface": "report", "path": "binding_surface_facts", "classification": "path_metadata"}, + {"surface": "report", "path": "binding_surface_diff", "classification": "path_metadata"}, {"surface": "report", "path": "capability_runtime_evidence", "classification": "path_metadata"}, {"surface": "report", "path": "capability_runtime_evidence.matched[].observed", "classification": "public_control_metadata"}, {"surface": "report", "path": "capability_runtime_evidence.unmatched[].observed", "classification": "public_control_metadata"}, @@ -46,6 +48,7 @@ {"surface": "report", "path": "loaded_plugins", "classification": "path_metadata"}, {"surface": "report", "path": "loaded_adapters", "classification": "path_metadata"}, {"surface": "report", "path": "tool_inventory", "classification": "credential_metadata"}, + {"surface": "report", "path": "tool_catalog", "classification": "credential_metadata"}, {"surface": "report", "path": "source_warnings", "classification": "free_text"}, {"surface": "report", "path": "agent_summary", "classification": "free_text"}, {"surface": "report", "path": "policy_audit", "classification": "free_text"}, diff --git a/docs/report-v1-consolidation-rc.md b/docs/report-v1-consolidation-rc.md index adeba993..48611882 100644 --- a/docs/report-v1-consolidation-rc.md +++ b/docs/report-v1-consolidation-rc.md @@ -2,7 +2,7 @@ > Status: **proposal** (no behavior change in this document). Target: > freeze a v1.0 report schema whose top-level surface stops growing. -> Written 2026-06; current runtime is `report_schema_version: "0.30"`. +> Written 2026-06; current runtime is `report_schema_version: "0.31"`. ## Problem diff --git a/examples/agent-protocol/expected/block-stop.json b/examples/agent-protocol/expected/block-stop.json index 179c5b37..e26ecde2 100644 --- a/examples/agent-protocol/expected/block-stop.json +++ b/examples/agent-protocol/expected/block-stop.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/examples/agent-protocol/expected/policy-bypass.json b/examples/agent-protocol/expected/policy-bypass.json index 2c500fc6..d3190c32 100644 --- a/examples/agent-protocol/expected/policy-bypass.json +++ b/examples/agent-protocol/expected/policy-bypass.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/examples/agent-protocol/expected/repair-after.json b/examples/agent-protocol/expected/repair-after.json index 2e31d0e4..8e263a9e 100644 --- a/examples/agent-protocol/expected/repair-after.json +++ b/examples/agent-protocol/expected/repair-after.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/examples/agent-protocol/expected/repair-before.json b/examples/agent-protocol/expected/repair-before.json index 231efcbe..b0a1e4ed 100644 --- a/examples/agent-protocol/expected/repair-before.json +++ b/examples/agent-protocol/expected/repair-before.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/llms-full.txt b/llms-full.txt index 993147a9..141d6121 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -164,9 +164,11 @@ expanding baselines or waivers, removing Shipgate CI, or weakening agent instructions. Verify-mode `SHIP-VERIFY-*` checks make those trust-root edits release-visible and route them to human review. Never invent or auto-fill an action effect or action authority declaration. -Contract v11 publishes these boundaries as `action_effect` and -`action_authority` in `do_not_auto_assert[]`; route semantic next actions to a -human and rerun verification after the reviewed declaration is supplied. +Never invent or auto-fill `agent_bindings` root, tool, or handoff declarations; +they are reviewed closed-world claims about deployed wiring. +Contract v13 publishes these boundaries as `action_effect`, `action_authority`, +and `agent_binding` in `do_not_auto_assert[]`; route binding and semantic next +actions to a human and rerun verification after the reviewed declaration is supplied. To reproduce the verify-native blocked refund PR demo without writing YAML: @@ -368,13 +370,13 @@ restate version archaeology here): - `findings[].{autofix_safe, requires_human_review, suggested_patch_kind, docs_url, provenance_kind, blocks_release}` - `findings[].policy_routing` (policy-pack owner/reviewer/approval routing metadata only; non-enforcing and not part of `evidence`) - `findings[].patches[]` (only when scan ran with `--suggest-patches`) -- `baseline.{matched_count, new_count, resolved_count}` · `tool_inventory[]` · `codex_plugin_surface` +- `baseline.{matched_count, new_count, resolved_count}` · root-reachable `tool_inventory[]` · full `tool_catalog[]` · `codex_plugin_surface` - `action_surface_facts` / `action_surface_diff` -- `release_decision.evidence_coverage.{semantic_coverage,evidence_gaps}` +- `release_decision.evidence_coverage.{binding_coverage,semantic_coverage,evidence_gaps}` - Audit envelopes: `release_decision.contribution_rules[]`, `policy_audit`, `privacy_audit`, `heuristics_filter` — explanatory, never a second gate -The current schema is [`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json). Emitted reports carry `report_schema_version: "0.30"`; v0.29 adds normalized effect/authority assessments and zero-tolerance semantic coverage. A `passed` result now requires complete, conflict-free static evidence for every in-scope action, evaluation of all applicable controls, and no policy condition requiring review. Every release decision explicitly carries `static_analysis_only: true`, `runtime_behavior_verified: false`, and `static_verdict_disclaimer`; packet §1 mirrors them. Semantic gaps are not Findings and cannot be suppressed or baselined. See [`docs/passed-verdict-contract.md`](docs/passed-verdict-contract.md) for the exact claim and [`docs/agent-contract-current.md`](docs/agent-contract-current.md) for version history. v0.29 remains frozen at [`docs/report-schema.v0.29.json`](docs/report-schema.v0.29.json). +The current schema is [`docs/report-schema.v0.31.json`](docs/report-schema.v0.31.json). Emitted reports carry `report_schema_version: "0.31"`; v0.31 separates the full tool catalog from the root-reachable capability surface and adds binding facts, diffs, and coverage. A `passed` result requires a complete root-reachable static binding graph plus complete, conflict-free identity, effect, and authority evidence for every reachable action, evaluation of all applicable controls, and no policy condition requiring review. Every release decision explicitly carries `static_analysis_only: true`, `runtime_behavior_verified: false`, and `static_verdict_disclaimer`; packet §1 mirrors them. Binding and semantic gaps are not Findings and cannot be suppressed or baselined. See [`docs/passed-verdict-contract.md`](docs/passed-verdict-contract.md) for the exact claim and [`docs/agent-contract-current.md`](docs/agent-contract-current.md) for version history. v0.30 remains frozen at [`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json). **Release gating signal**: prefer `release_decision.decision` (`"blocked" | "review_required" | "insufficient_evidence" | "passed"`) over `summary.status`. The new field is **baseline-aware** — a baseline-matched critical surfaces in `release_decision.review_items` (accepted debt), not `release_decision.blockers`. `summary.status` stays baseline-blind for v0.7 compatibility, so a baseline-matched-only critical produces both `summary.status = "release_blockers_detected"` AND `release_decision.decision = "review_required"` (intentional divergence — see [STABILITY.md](STABILITY.md#release_decisiondecision-vs-summarystatus)). `insufficient_evidence` (added v0.14) signals that the scan saw too many low-confidence tools or source-loader warnings to be trustworthy; consumers that switch on the enum must fall back to `review_required` for unknown future values. @@ -454,7 +456,7 @@ validation and [`docs/manifest-v0.1.md`](docs/manifest-v0.1.md) for prose. ### Where is the report schema? Parse `agents-shipgate-reports/report.json` and validate against -[`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json) (current). +[`docs/report-schema.v0.31.json`](docs/report-schema.v0.31.json) (current). Older reports (`report_schema_version: "0.10"`) validate against the frozen [`docs/report-schema.v0.10.json`](docs/report-schema.v0.10.json). Do not scrape Markdown when JSON is available. @@ -492,7 +494,8 @@ For the short, current statement of "which fields to read", see [`docs/agent-con | What | Path | Stable | |---|---|---| | Manifest schema | [`docs/manifest-v0.1.json`](docs/manifest-v0.1.json) | `0.1` | -| Report schema (current) | [`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json) | `0.30` | +| Report schema (current) | [`docs/report-schema.v0.31.json`](docs/report-schema.v0.31.json) | `0.31` | +| Report schema (v0.30 frozen reference) | [`docs/report-schema.v0.30.json`](docs/report-schema.v0.30.json) | `0.30` | | Report schema (v0.29 frozen reference) | [`docs/report-schema.v0.29.json`](docs/report-schema.v0.29.json) | `0.29` | | Report schema (v0.28 frozen reference) | [`docs/report-schema.v0.28.json`](docs/report-schema.v0.28.json) | `0.28` | | Report schema (v0.27 frozen reference) | [`docs/report-schema.v0.27.json`](docs/report-schema.v0.27.json) | `0.27` | @@ -520,12 +523,13 @@ For the short, current statement of "which fields to read", see [`docs/agent-con | Report schema (v0.8 frozen reference) | [`docs/report-schema.v0.8.json`](docs/report-schema.v0.8.json) | `0.8` | | Report schema (v0.7 frozen reference) | [`docs/report-schema.v0.7.json`](docs/report-schema.v0.7.json) | `0.7` | | Report schema (v0.6 frozen reference) | [`docs/report-schema.v0.6.json`](docs/report-schema.v0.6.json) | `0.6` | -| Packet schema (Release Evidence Packet, latest) | [`docs/packet-schema.v0.9.json`](docs/packet-schema.v0.9.json) | `0.9` | +| Packet schema (Release Evidence Packet, latest) | [`docs/packet-schema.v0.10.json`](docs/packet-schema.v0.10.json) | `0.10` | +| Packet schema (v0.9 frozen reference) | [`docs/packet-schema.v0.9.json`](docs/packet-schema.v0.9.json) | `0.9` | | Packet schema (v0.8 frozen reference) | [`docs/packet-schema.v0.8.json`](docs/packet-schema.v0.8.json) | `0.8` | | Packet schema (v0.7 frozen reference) | [`docs/packet-schema.v0.7.json`](docs/packet-schema.v0.7.json) | `0.7` | | Capability standard | [`docs/capability-standard.md`](docs/capability-standard.md) | `0.3` | -| Capability lock schema | [`docs/capability-lock-schema.v0.4.json`](docs/capability-lock-schema.v0.4.json) | `0.4` | -| Capability lock diff schema | [`docs/capability-lock-diff-schema.v0.5.json`](docs/capability-lock-diff-schema.v0.5.json) | `0.5` | +| Capability lock schema | [`docs/capability-lock-schema.v0.5.json`](docs/capability-lock-schema.v0.5.json) | `0.5` | +| Capability lock diff schema | [`docs/capability-lock-diff-schema.v0.6.json`](docs/capability-lock-diff-schema.v0.6.json) | `0.6` | | Governance benchmark catalog schema | [`docs/governance-benchmark-catalog-schema.v0.2.json`](docs/governance-benchmark-catalog-schema.v0.2.json) | `0.2` | | Governance benchmark result schema | [`docs/governance-benchmark-result-schema.v0.2.json`](docs/governance-benchmark-result-schema.v0.2.json) | `0.2` | | Check catalog | [`docs/checks.json`](docs/checks.json) | regenerated each release | @@ -585,7 +589,7 @@ NOT prove. Use `--no-packet` / `--packet-format` on `scan`, and `agents-shipgate evidence-packet --from ` to re-render. The full packet contract (fixed sections, disclaimers, `evidence_matrix` rules) lives in -[STABILITY.md §Release Evidence Packet](STABILITY.md#release-evidence-packet-v09) +[STABILITY.md §Release Evidence Packet](STABILITY.md#release-evidence-packet-v010) and [`docs/agent-contract-current.md`](docs/agent-contract-current.md#read-these-for-release-review). Exit codes (stable): @@ -1003,9 +1007,10 @@ Verify the installed CLI contract locally before relying on hard-coded docs: agents-shipgate contract --json ``` -Runtime contract v12 makes `passed` an evidence-backed static verdict and -publishes report v0.30, packet v0.9, capability standard v0.3, capability lock -v0.4, and capability-lock diff v0.5. Report fields are additive over v0.29, +Runtime contract v13 proves the root-reachable agent-to-tool graph before +evaluating capabilities and publishes report v0.31, packet v0.10, capability +standard v0.4, capability lock v0.5, and capability-lock diff v0.6. Report +fields are additive over v0.30, but the pre-1.0 meaning of `passed` is deliberately stricter. v10 added `verify_required` to `agent_result_control_fields` and to the boundary result. The runtime contract also exposes the local agent command spec: @@ -1033,25 +1038,25 @@ Downstream repos generated with `.shipgate/agent-contract.json`. - Latest release: `v0.15.0` -- In-tree runtime: `0.16.0b1` — see [pyproject.toml](../pyproject.toml) -- Runtime contract: `12` -- Current report schema: `0.30` — [`docs/report-schema.v0.30.json`](report-schema.v0.30.json) -- Current packet schema: `0.9` — [`docs/packet-schema.v0.9.json`](packet-schema.v0.9.json) +- In-tree runtime: `0.16.0b2` — see [pyproject.toml](../pyproject.toml) +- Runtime contract: `13` +- Current report schema: `0.31` — [`docs/report-schema.v0.31.json`](report-schema.v0.31.json) +- Current packet schema: `0.10` — [`docs/packet-schema.v0.10.json`](packet-schema.v0.10.json) - Current verifier schema: `0.2` — [`docs/verifier-schema.v0.2.json`](verifier-schema.v0.2.json) - Current verify-run schema: `shipgate.verify_run/v1` — [`docs/verify-run-schema.v1.json`](verify-run-schema.v1.json) - Current agent handoff schema: `shipgate.agent_handoff/v2` — [`docs/agent-handoff-schema.v2.json`](agent-handoff-schema.v2.json) - Current Codex boundary result schema: `shipgate.codex_boundary_result/v1` — [`docs/codex-boundary-result-schema.v1.json`](codex-boundary-result-schema.v1.json) - Current preflight schema: `0.2` — [`docs/preflight-schema.v0.2.json`](preflight-schema.v0.2.json) -- Current capability standard: `0.3` — [`docs/capability-standard.md`](capability-standard.md) -- Current capability lock schema: `0.4` — [`docs/capability-lock-schema.v0.4.json`](capability-lock-schema.v0.4.json) -- Current capability lock diff schema: `0.5` — [`docs/capability-lock-diff-schema.v0.5.json`](capability-lock-diff-schema.v0.5.json) +- Current capability standard: `0.4` — [`docs/capability-standard.md`](capability-standard.md) +- Current capability lock schema: `0.5` — [`docs/capability-lock-schema.v0.5.json`](capability-lock-schema.v0.5.json) +- Current capability lock diff schema: `0.6` — [`docs/capability-lock-diff-schema.v0.6.json`](capability-lock-diff-schema.v0.6.json) - Current attestation schema: `0.4` — [`docs/attestation-schema.v0.4.json`](attestation-schema.v0.4.json) - Current registry schema: `0.3` — [`docs/registry-schema.v0.3.json`](registry-schema.v0.3.json) - Current org evidence bundle schema: `shipgate.org_evidence_bundle/v1` — [`docs/org-evidence-bundle-schema.v1.json`](org-evidence-bundle-schema.v1.json) - Current host-grants inventory schema: `0.1` — [`docs/host-grants-inventory-schema.v0.1.json`](host-grants-inventory-schema.v0.1.json) - Current governance benchmark catalog schema: `0.2` — [`docs/governance-benchmark-catalog-schema.v0.2.json`](governance-benchmark-catalog-schema.v0.2.json) - Current governance benchmark result schema: `0.2` — [`docs/governance-benchmark-result-schema.v0.2.json`](governance-benchmark-result-schema.v0.2.json) -- Frozen-reference report schemas: frozen [`v0.29`](report-schema.v0.29.json), frozen [`v0.28`](report-schema.v0.28.json), frozen [`v0.27`](report-schema.v0.27.json), frozen [`v0.26`](report-schema.v0.26.json), frozen [`v0.25`](report-schema.v0.25.json), frozen [`v0.24`](report-schema.v0.24.json), frozen [`v0.23`](report-schema.v0.23.json), frozen [`v0.22`](report-schema.v0.22.json), frozen [`v0.21`](report-schema.v0.21.json), frozen [`v0.20`](report-schema.v0.20.json), frozen [`v0.19`](report-schema.v0.19.json), frozen [`v0.18`](report-schema.v0.18.json), frozen [`v0.17`](report-schema.v0.17.json), frozen [`v0.16`](report-schema.v0.16.json), frozen [`v0.15`](report-schema.v0.15.json), frozen [`v0.14`](report-schema.v0.14.json), frozen [`v0.13`](report-schema.v0.13.json), frozen [`v0.12`](report-schema.v0.12.json), frozen [`v0.11`](report-schema.v0.11.json), frozen [`v0.10`](report-schema.v0.10.json), frozen [`v0.9`](report-schema.v0.9.json), frozen [`v0.8`](report-schema.v0.8.json), frozen [`v0.7`](report-schema.v0.7.json), frozen [`v0.6`](report-schema.v0.6.json), older +- Frozen-reference report schemas: frozen [`v0.30`](report-schema.v0.30.json), frozen [`v0.29`](report-schema.v0.29.json), frozen [`v0.28`](report-schema.v0.28.json), frozen [`v0.27`](report-schema.v0.27.json), frozen [`v0.26`](report-schema.v0.26.json), frozen [`v0.25`](report-schema.v0.25.json), frozen [`v0.24`](report-schema.v0.24.json), frozen [`v0.23`](report-schema.v0.23.json), frozen [`v0.22`](report-schema.v0.22.json), frozen [`v0.21`](report-schema.v0.21.json), frozen [`v0.20`](report-schema.v0.20.json), frozen [`v0.19`](report-schema.v0.19.json), frozen [`v0.18`](report-schema.v0.18.json), frozen [`v0.17`](report-schema.v0.17.json), frozen [`v0.16`](report-schema.v0.16.json), frozen [`v0.15`](report-schema.v0.15.json), frozen [`v0.14`](report-schema.v0.14.json), frozen [`v0.13`](report-schema.v0.13.json), frozen [`v0.12`](report-schema.v0.12.json), frozen [`v0.11`](report-schema.v0.11.json), frozen [`v0.10`](report-schema.v0.10.json), frozen [`v0.9`](report-schema.v0.9.json), frozen [`v0.8`](report-schema.v0.8.json), frozen [`v0.7`](report-schema.v0.7.json), frozen [`v0.6`](report-schema.v0.6.json), older - Frozen-reference packet schemas live in [`docs/INDEX.md`](INDEX.md#reference). - Frozen experimental capability lock and governance benchmark result schemas live in [`docs/INDEX.md`](INDEX.md#reference). @@ -1497,8 +1502,8 @@ For reviewer-shaped output, also read the **Release Evidence Packet** at `[pdf]` extras are installed). The packet is a supporting/provisional reviewer projection, not a second gate. Packet outputs are redacted by the same default privacy layer as the report. The packet has fixed reviewer sections governed by -[`docs/packet-schema.v0.9.json`](packet-schema.v0.9.json) — see -[STABILITY.md §Release Evidence Packet](../STABILITY.md#release-evidence-packet-v09). +[`docs/packet-schema.v0.10.json`](packet-schema.v0.10.json) — see +[STABILITY.md §Release Evidence Packet](../STABILITY.md#release-evidence-packet-v010). Packet schema `0.9` carries the report's evidence-backed semantic coverage and gap remediation contract. Packet §1 also mirrors `static_analysis_only=true`, `runtime_behavior_verified=false`, and @@ -1551,9 +1556,9 @@ Companion prompt: [`prompts/explain-finding-to-user.md`](../prompts/explain-find - [STABILITY.md](../STABILITY.md) — full alpha stability contract. Source of truth for everything above. - [AGENTS.md](../AGENTS.md) — agent-facing instructions: install, run, single-turn flow, error semantics. -- [`docs/report-schema.v0.30.json`](report-schema.v0.30.json) — machine-validatable JSON Schema for the current report. +- [`docs/report-schema.v0.31.json`](report-schema.v0.31.json) — machine-validatable JSON Schema for the current report. - [`docs/privacy.md`](privacy.md) and [`docs/report-sensitive-fields.json`](report-sensitive-fields.json) — default redaction behavior and sensitive-field inventory. -- [`docs/packet-schema.v0.9.json`](packet-schema.v0.9.json) — machine-validatable JSON Schema for the current packet. +- [`docs/packet-schema.v0.10.json`](packet-schema.v0.10.json) — machine-validatable JSON Schema for the current packet. - [`docs/checks.json`](checks.json) — check catalog, including `mvp_tier` for MVP/readiness triage. ## See also @@ -3046,7 +3051,7 @@ from the hash so toggling `--suggest-patches` doesn't shift it. - [`checks.md`](checks.md) — full check catalog with rationale. - [`minimal-real-configs.md`](minimal-real-configs.md) — per-framework minimal manifests to build from. -- [`report-schema.v0.30.json`](report-schema.v0.30.json) — current JSON +- [`report-schema.v0.31.json`](report-schema.v0.31.json) — current JSON Schema for `report.json`. - [`AGENTS.md`](../AGENTS.md) — top-level agent instructions, install, trigger table. diff --git a/llms.txt b/llms.txt index 5e4e2184..dd103bd5 100644 --- a/llms.txt +++ b/llms.txt @@ -13,7 +13,7 @@ - Publisher URL: https://threemoonslab.com/ - License: Apache-2.0 - Latest public release: v0.15.0 -- Current source-tree runtime: 0.16.0b1 (contract 11; pre-release) +- Current source-tree runtime: 0.16.0b2 (contract 13; pre-release) - Canonical repository: https://github.com/ThreeMoonsLab/agents-shipgate - Do not use: Agent Shipcheck, Agent Shipgate, agents shipgate, Agents-Shipgate @@ -60,11 +60,11 @@ - Markdown report: `agents-shipgate-reports/report.md`. - JSON report: `agents-shipgate-reports/report.json`. -- JSON report schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/report-schema.v0.30.json +- JSON report schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/report-schema.v0.31.json - Evidence-backed passed contract: https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/passed-verdict-contract.md - Machine verdict boundary: `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, plus `static_verdict_disclaimer`; packet §1 mirrors these fields. - Release Evidence Packet (Markdown / JSON / HTML, optional PDF): `agents-shipgate-reports/packet.{md,json,html}`. -- Packet schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/packet-schema.v0.9.json +- Packet schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/packet-schema.v0.10.json - SARIF report: `agents-shipgate-reports/report.sarif`. - Verifier orchestration record (ongoing-PR verify): `agents-shipgate-reports/verifier.json`. - Agent handoff artifact (preferred compact agent projection for verify): `agents-shipgate-reports/agent-handoff.json`. @@ -152,9 +152,9 @@ - Zero-install detector (stdlib-only Python; same structural verdict as `agents-shipgate detect --json` — emits the canonical `DetectResult` fields plus `script_version`, but NOT the CLI's `diagnostics` or `next_actions` arrays): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/tools/shipgate-detect.py - Zero-install paths overview (single-file detector, uvx, GitHub Action): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/zero-install.md - Manifest schema: https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/manifest-v0.1.json -- Report schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/report-schema.v0.30.json +- Report schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/report-schema.v0.31.json - Privacy/redaction docs: https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/privacy.md -- Packet schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/packet-schema.v0.9.json +- Packet schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/packet-schema.v0.10.json - Preflight schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/preflight-schema.v0.2.json - Capability standard: https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/capability-standard.md - Capability lock schema (current): https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/capability-lock-schema.v0.4.json diff --git a/plugins/agents-shipgate/.codex-plugin/plugin.json b/plugins/agents-shipgate/.codex-plugin/plugin.json index afa86626..e3c690ea 100644 --- a/plugins/agents-shipgate/.codex-plugin/plugin.json +++ b/plugins/agents-shipgate/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agents-shipgate", - "version": "0.16.0b1", + "version": "0.16.0b2", "description": "Run Agents Shipgate Tool-Use Readiness workflows from Codex.", "author": { "name": "Three Moons Lab", diff --git a/plugins/agents-shipgate/skills/agents-shipgate/SKILL.md b/plugins/agents-shipgate/skills/agents-shipgate/SKILL.md index 80107eb8..7c8caf80 100644 --- a/plugins/agents-shipgate/skills/agents-shipgate/SKILL.md +++ b/plugins/agents-shipgate/skills/agents-shipgate/SKILL.md @@ -21,7 +21,7 @@ Do not use it for general linting, runtime monitoring, evals, model-output quali 6. For local agent control, run `shipgate check --agent codex --workspace . --format codex-boundary-json` and read the stdout `shipgate.codex_boundary_result/v1` object. Switch on `decision`; follow `first_next_action`, `repair`, and `human_review`. 7. Before editing `shipgate.yaml`, Shipgate CI, AGENTS/CLAUDE/Cursor rules, policy packs, baselines, waivers, suppressions, Codex hooks/config, Codex plugin manifests, `.mcp.json`, `.app.json`, or `SKILL.md`, plan to run `agents-shipgate verify` before completion and route trust-root review to a human when the verifier requires it. 8. For full PR verification, read `agents-shipgate-reports/agent-handoff.json` first, then `verifier.json` for detailed controller state, then `verify-run.json` for reproducibility metadata, then `report.json` for reviewer detail; `report.json.release_decision.decision` remains the release gate. -9. Auto-apply only high-confidence safe patches. Do not auto-assert action effect, action authority, approval, confirmation, idempotency, broad-scope, prohibited-action, or runtime-trace evidence. +9. Auto-apply only high-confidence safe patches. Do not auto-assert action effect, action authority, agent bindings, approval, confirmation, idempotency, broad-scope, prohibited-action, or runtime-trace evidence. 10. Ensure `.gitignore` covers `agents-shipgate-reports/` before committing. ## Fast Paths diff --git a/plugins/claude-code/.claude-plugin/plugin.json b/plugins/claude-code/.claude-plugin/plugin.json index a0b822fc..169bac34 100644 --- a/plugins/claude-code/.claude-plugin/plugin.json +++ b/plugins/claude-code/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agents-shipgate", - "version": "0.16.0b1", + "version": "0.16.0b2", "description": "Run Agents Shipgate Tool-Use Readiness workflows from Claude Code. Skill-only: the plugin supplies the agents-shipgate skill and the /agents-shipgate:shipgate command; the scanner runs through the agents-shipgate CLI installed in the local environment (pipx install agents-shipgate, contract v10 or newer). For the deterministic hooks (trigger check after edits, full verify at Stop), run: agents-shipgate install-hooks --target claude-code --write.", "author": { "name": "Three Moons Lab", diff --git a/plugins/claude-code/skills/agents-shipgate/SKILL.md b/plugins/claude-code/skills/agents-shipgate/SKILL.md index f072f512..00126def 100644 --- a/plugins/claude-code/skills/agents-shipgate/SKILL.md +++ b/plugins/claude-code/skills/agents-shipgate/SKILL.md @@ -76,9 +76,9 @@ For non-GitHub CI (GitLab, CircleCI, Jenkins, Azure Pipelines, Buildkite, Bitbuc - **Installed CLI contract**: when available, run `agents-shipgate contract --json` to verify local schema versions, capability/research surfaces, `release_decision.decision`, and manual-review signal fields. Older installs should use [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md) or upgrade before automating against the local contract command. - **Verifier JSON**: `verifier_schema_version: "0.2"`. Read `merge_verdict`, `can_merge_without_human`, `first_next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. - **Verify run JSON**: `verify-run.json` uses `schema_version: "shipgate.verify_run/v1"` and records stable run identity, subject refs, input hashes, outcome, and artifact hashes. It is the reproducibility artifact for `verify`; do not treat it as a second gate. -- **Report JSON**: `report_schema_version: "0.30"`. Read `release_decision.decision` first. A `passed` decision now requires complete, conflict-free static effect and authority evidence for every in-scope action, evaluation of all applicable controls, and no policy condition requiring review; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries; packet §1 mirrors them. Read `release_decision.evidence_coverage.semantic_coverage` and `identity_coverage`, then work every `evidence_gaps[].next_action` in order. Semantic gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; effect and authority declarations are human assertions and must never be auto-written. Use `agent_summary` / `findings[].agent_action` for routing and `reviewer_summary` for human review. To inspect heuristic findings without changing gate behavior, use `agents-shipgate findings --from agents-shipgate-reports/report.json --provenance-kind keyword_heuristic,regex_heuristic,runtime_trace --json`. Do not gate on legacy, baseline-blind `summary.status`. The current schema is [`docs/report-schema.v0.30.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.30.json); v0.29 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md#read-these-first-for-release-gating) and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). - These evidence-backed fields require runtime contract v12. A contract-v11 CLI emits the frozen v0.29 report and must not be described using the v0.30 identity claim. -- **Release Evidence Packet**: `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras) is a supporting/provisional reviewer artifact. Packet v0.9 projects semantic coverage and gap remediation under the release decision; it never creates a second gate. See [`docs/packet-schema.v0.9.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/packet-schema.v0.9.json) and [STABILITY.md §Release Evidence Packet](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md#release-evidence-packet-v09). +- **Report JSON**: `report_schema_version: "0.31"`. Read `release_decision.decision` first. A `passed` decision requires a complete root-reachable static binding graph plus complete, conflict-free identity, effect, and authority evidence for every reachable action; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries. Read `release_decision.evidence_coverage.binding_coverage`, `semantic_coverage`, and `identity_coverage`, then work every `evidence_gaps[].next_action` in order. Binding and semantic gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; binding, effect, and authority declarations are human assertions and must never be auto-written. Use `tool_catalog[]` for diagnostics and `tool_inventory[]` for the proven reachable surface. The current schema is [`docs/report-schema.v0.31.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.31.json); v0.30 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md#read-these-first-for-release-gating) and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). + These binding-backed fields require runtime contract v13. A contract-v12 CLI emits the frozen v0.30 report and must not be described using the v0.31 binding claim. +- **Release Evidence Packet**: `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras) is a supporting/provisional reviewer artifact. Packet v0.10 projects binding and semantic coverage under the release decision; it never creates a second gate. See [`docs/packet-schema.v0.10.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/packet-schema.v0.10.json) and [STABILITY.md §Release Evidence Packet](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md#release-evidence-packet-v010). - **Capability standard**: `agents-shipgate capability export` emits a stable static capability lock (`capability_lock_schema_version: "0.4"`) and `agents-shipgate capability diff` emits a stable semantic diff (`capability_lock_diff_schema_version: "0.5"`). These artifacts implement capability standard v0.3, are supporting/provisional and non-gating, exclude runtime trace evidence, and are documented in [`docs/capability-standard.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/capability-standard.md). - **Governance benchmark**: `benchmark/agent-pr-governance/cases.yaml` and `scripts/run_governance_benchmark.py` are the stable research benchmark substrate (`governance_benchmark_result_schema_version: "0.2"`), not a release gate. See [`docs/governance-benchmark.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/governance-benchmark.md). - **Single source of truth for the contract**: [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md). When the schema bumps, that file updates first. diff --git a/pyproject.toml b/pyproject.toml index 6b7aba15..54e527a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agents-shipgate" -version = "0.16.0b1" +version = "0.16.0b2" description = "The deterministic merge gate for AI-generated agent capability changes. Agent release readiness for tool-using AI agents. CLI + GitHub Action. Scans MCP, OpenAPI, OpenAI Agents SDK, Anthropic, Google ADK, LangChain, CrewAI, OpenAI API, Codex config, Codex plugin, n8n." readme = "README.md" requires-python = ">=3.12" diff --git a/samples/agent_weakens_gate/shipgate.yaml b/samples/agent_weakens_gate/shipgate.yaml index 205aa6f0..efd87d13 100644 --- a/samples/agent_weakens_gate/shipgate.yaml +++ b/samples/agent_weakens_gate/shipgate.yaml @@ -18,6 +18,15 @@ tool_sources: type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: docs.lookup, source_id: docs_tools} + handoffs: [] + reason: reviewed local fixture binding + permissions: scopes: - docs:read diff --git a/samples/ai_generated_refund_pr/shipgate.yaml b/samples/ai_generated_refund_pr/shipgate.yaml index 5e026308..924d6c54 100644 --- a/samples/ai_generated_refund_pr/shipgate.yaml +++ b/samples/ai_generated_refund_pr/shipgate.yaml @@ -19,6 +19,16 @@ tool_sources: path: tools.json trust: internal +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: support.search_kb, source_id: support_tools} + - {tool: stripe.create_refund, source_id: support_tools} + handoffs: [] + reason: reviewed verifier demo binding + permissions: scopes: - support:kb:read diff --git a/samples/clean_read_only_agent/shipgate.yaml b/samples/clean_read_only_agent/shipgate.yaml index fa393972..5ad5555a 100644 --- a/samples/clean_read_only_agent/shipgate.yaml +++ b/samples/clean_read_only_agent/shipgate.yaml @@ -16,6 +16,15 @@ tool_sources: type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: docs.lookup, source_id: docs_tools} + handoffs: [] + reason: reviewed local fixture binding + permissions: scopes: - docs:read diff --git a/samples/hitl_evidence_agent/shipgate.yaml b/samples/hitl_evidence_agent/shipgate.yaml index 166520dd..9b352cf1 100644 --- a/samples/hitl_evidence_agent/shipgate.yaml +++ b/samples/hitl_evidence_agent/shipgate.yaml @@ -19,6 +19,15 @@ tool_sources: type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: issue_refund, source_id: refund_tools} + handoffs: [] + reason: reviewed HITL fixture binding + validation: mode: human_in_the_loop target_review_posture: limited_auto_approval diff --git a/samples/hitl_evidence_covered_agent/shipgate.yaml b/samples/hitl_evidence_covered_agent/shipgate.yaml index 182eb2e3..41e11f07 100644 --- a/samples/hitl_evidence_covered_agent/shipgate.yaml +++ b/samples/hitl_evidence_covered_agent/shipgate.yaml @@ -19,6 +19,15 @@ tool_sources: type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: issue_refund, source_id: refund_tools} + handoffs: [] + reason: reviewed HITL fixture binding + validation: mode: human_in_the_loop target_review_posture: limited_auto_approval diff --git a/samples/openai_agents_sdk_agent/README.md b/samples/openai_agents_sdk_agent/README.md index 45ba19a4..a37ce9c4 100644 --- a/samples/openai_agents_sdk_agent/README.md +++ b/samples/openai_agents_sdk_agent/README.md @@ -1,7 +1,8 @@ OpenAI Agents SDK static extraction fixture with a directory of Python tools. This fixture exercises the conservative `openai_agents_sdk` adapter path: it -parses local Python files with AST, extracts `@function_tool` declarations, and +parses local Python files with AST, extracts `@function_tool` declarations and +the literal `support_assistant = Agent(..., tools=[...])` binding, and does not import user code. Directory scanning is non-recursive: only immediate `*.py` files under `agents/` are scanned in sorted order. `agents/dynamic_tools.py` intentionally contains a runtime factory that is diff --git a/samples/openai_agents_sdk_agent/agents/root.py b/samples/openai_agents_sdk_agent/agents/root.py new file mode 100644 index 00000000..8bf4063c --- /dev/null +++ b/samples/openai_agents_sdk_agent/agents/root.py @@ -0,0 +1,8 @@ +from agents import Agent +from case_tools import lookup_case +from reply_tools import render_reply + +support_assistant = Agent( + name="support-sdk-assistant", + tools=[lookup_case, render_reply], +) diff --git a/samples/openai_agents_sdk_agent/shipgate.yaml b/samples/openai_agents_sdk_agent/shipgate.yaml index 6ea56d3d..b87b1e1b 100644 --- a/samples/openai_agents_sdk_agent/shipgate.yaml +++ b/samples/openai_agents_sdk_agent/shipgate.yaml @@ -9,6 +9,7 @@ agent: type: openai-agents language: python entrypoint: agents + object: support_assistant static_extract: true deep_import: false declared_purpose: diff --git a/samples/simple_anthropic_agent/shipgate.yaml b/samples/simple_anthropic_agent/shipgate.yaml index 6c668924..84554400 100644 --- a/samples/simple_anthropic_agent/shipgate.yaml +++ b/samples/simple_anthropic_agent/shipgate.yaml @@ -23,6 +23,16 @@ anthropic: policy_rules: - path: policies/anthropic-policy.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: create_refund} + - {tool: get_help_article} + handoffs: [] + reason: reviewed Anthropic Messages fixture binding + action_surface: actions: - tool: create_refund diff --git a/samples/simple_crewai_agent/expected/report.json b/samples/simple_crewai_agent/expected/report.json index fc8d9443..aaeb1eee 100644 --- a/samples/simple_crewai_agent/expected/report.json +++ b/samples/simple_crewai_agent/expected/report.json @@ -1,7 +1,7 @@ { "schema_version": "0.1", - "report_schema_version": "0.30", - "run_id": "agents_shipgate_136deceabf653b25", + "report_schema_version": "0.31", + "run_id": "agents_shipgate_a62c844f928e0aba", "manifest_dir": "/samples/simple_crewai_agent", "project": { "name": "simple-crewai-agent" @@ -132,6 +132,15 @@ "ambiguous_name_count": 0, "gap_count": 0, "reason_counts": {} + }, + "binding_coverage": { + "total_catalog_tools": 3, + "reachable_tools": 3, + "possible_tools": 0, + "unbound_tools": 0, + "pass_eligible": true, + "gap_count": 0, + "reason_counts": {} } }, "baseline_delta": { @@ -215,6 +224,32 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -474,7 +509,7 @@ ] }, "action_surface_facts": { - "snapshot_version": "0.2", + "snapshot_version": "0.3", "actions": [ { "action_id": "agent:simple-crewai-agent/support-case-crew:action_v2_56f5d7d377130876f26986bdadd25cd1ad47dd0138cba6da8dc11e7d06fe30e6", @@ -536,6 +571,32 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -694,6 +755,32 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -852,6 +939,32 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -977,8 +1090,89 @@ "modified": [], "notes": [ "No action-surface comparison source was provided." + ] + }, + "binding_surface_facts": { + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "status": "structural", + "agents": [ + { + "agent_id": "agent_v1:71a7997718e73610b0704464", + "name": "researcher", + "source_id": "crewai_agent", + "source_ref": "crew.py", + "source_pointer": "crew.py:31" + }, + { + "agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "name": "crew", + "source_id": "crewai_agent", + "source_ref": "crew.py", + "source_pointer": "crew.py:38" + } + ], + "tool_edges": [ + { + "agent_id": "agent_v1:71a7997718e73610b0704464", + "tool_id": "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "complete": true + }, + { + "agent_id": "agent_v1:71a7997718e73610b0704464", + "tool_id": "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "complete": true + }, + { + "agent_id": "agent_v1:71a7997718e73610b0704464", + "tool_id": "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "complete": true + } + ], + "handoff_edges": [ + { + "source_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "target_agent_id": "agent_v1:71a7997718e73610b0704464", + "edge_type": "subagent", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:38", + "complete": true + } + ], + "reachable_tool_ids": [ + "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638" ], - "tool_id": null + "possible_tool_ids": [], + "unbound_tool_ids": [], + "issues": [], + "pass_eligible": true + }, + "binding_surface_diff": { + "enabled": false, + "base_report_schema_version": null, + "added_reachable_tool_ids": [], + "removed_reachable_tool_ids": [], + "added_handoffs": [], + "removed_handoffs": [], + "notes": [] }, "capability_runtime_evidence": { "enabled": false, @@ -1062,8 +1256,8 @@ "Declare operation-specific auth scopes for FileReadTool, or explicitly declare anonymous authority when the operation requires no credentials." ], "generated_reports": { - "markdown": "expected/report.md", - "json": "expected/report.json" + "markdown": "/private/tmp/shipgate-golden-simple_crewai_agent/report.md", + "json": "/private/tmp/shipgate-golden-simple_crewai_agent/report.json" }, "loaded_policy_packs": [], "loaded_plugins": [], @@ -1134,6 +1328,32 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -1198,6 +1418,32 @@ "issues": [] }, "pass_eligible": true + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true } }, { @@ -1265,6 +1511,32 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -1329,6 +1601,32 @@ "issues": [] }, "pass_eligible": false + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true } }, { @@ -1396,6 +1694,32 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -1460,6 +1784,583 @@ "issues": [] }, "pass_eligible": true + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + } + } + ], + "tool_catalog": [ + { + "tool_id": "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "name": "lookup_case", + "provider": "crewai_agent", + "observation_ids": [ + "obs_v1_75cdf97835473a87a94cced0791d82e93653eb17faa022f9fe39a5156b4de1be", + "obs_v1_d80a6855053ad21b53e7781eb52851609ab5e62b9cdde3f3e17bb9bfd16d260c" + ], + "source_type": "crewai_inventory", + "source_ref": "inventories/tools.json", + "source_path": "inventories/tools.json", + "source_start_line": null, + "source_pointer": "/tools/0", + "risk_tags": [ + "read_only" + ], + "risk_tag_confidence": { + "read_only": "high" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "read", + "identity": { + "tool_id": "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "status": "declared", + "provider": "crewai_agent", + "binding_id": "crewai_lookup_case", + "primary_observation_id": "obs_v1_d80a6855053ad21b53e7781eb52851609ab5e62b9cdde3f3e17bb9bfd16d260c", + "observation_ids": [ + "obs_v1_75cdf97835473a87a94cced0791d82e93653eb17faa022f9fe39a5156b4de1be", + "obs_v1_d80a6855053ad21b53e7781eb52851609ab5e62b9cdde3f3e17bb9bfd16d260c" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_75cdf97835473a87a94cced0791d82e93653eb17faa022f9fe39a5156b4de1be", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "crew.py", + "evidence": { + "source_type": "crewai_class_tool", + "source_id": "crewai_agent", + "native_locator": "crew.py#lookup_case" + } + }, + { + "dimension": "identity", + "value": "obs_v1_d80a6855053ad21b53e7781eb52851609ab5e62b9cdde3f3e17bb9bfd16d260c", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "/tools/0", + "evidence": { + "source_type": "crewai_inventory", + "source_id": "crewai_inventory:inventories/tools.json", + "native_locator": "inventories/tools.json#lookup_case" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='lookup_case'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/tools/0", + "evidence": { + "readOnlyHint": true + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "none", + "auth_type": null, + "credential_mode": null, + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='lookup_case'].authority", + "evidence": { + "auth_type": null, + "credential_mode": null, + "scopes": [] + } + }, + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_authority", + "source_pointer": "/tools/0", + "evidence": { + "auth_type": null, + "scopes": [], + "alternative_count": 0 + } + } + ], + "issues": [] + }, + "pass_eligible": true + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_8bb9341b31f3731d4ef2337cde788263775c2f388329654c2bc8a8b521170ed1", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + } + }, + { + "tool_id": "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "name": "FileReadTool", + "provider": "crewai_agent", + "observation_ids": [ + "obs_v1_91dc9ef11aaa2cd63b91fb9e90b6607ecaf5e678f470c54d3520feb74c93aea8", + "obs_v1_d47ab93f36b5308c7f1fd05073fdca72147fe034a6ec7da6470753270c29bd32" + ], + "source_type": "crewai_inventory", + "source_ref": "inventories/tools.json", + "source_path": "inventories/tools.json", + "source_start_line": null, + "source_pointer": "/tools/2", + "risk_tags": [ + "read_only" + ], + "risk_tag_confidence": { + "read_only": "high" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "read", + "identity": { + "tool_id": "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "status": "declared", + "provider": "crewai_agent", + "binding_id": "crewai_file_read", + "primary_observation_id": "obs_v1_91dc9ef11aaa2cd63b91fb9e90b6607ecaf5e678f470c54d3520feb74c93aea8", + "observation_ids": [ + "obs_v1_91dc9ef11aaa2cd63b91fb9e90b6607ecaf5e678f470c54d3520feb74c93aea8", + "obs_v1_d47ab93f36b5308c7f1fd05073fdca72147fe034a6ec7da6470753270c29bd32" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_91dc9ef11aaa2cd63b91fb9e90b6607ecaf5e678f470c54d3520feb74c93aea8", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "/tools/2", + "evidence": { + "source_type": "crewai_inventory", + "source_id": "crewai_inventory:inventories/tools.json", + "native_locator": "inventories/tools.json#FileReadTool" + } + }, + { + "dimension": "identity", + "value": "obs_v1_d47ab93f36b5308c7f1fd05073fdca72147fe034a6ec7da6470753270c29bd32", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "crew.py", + "evidence": { + "source_type": "crewai_prebuilt_tool", + "source_id": "crewai_agent", + "native_locator": "crew.py#FileReadTool" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='FileReadTool'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/tools/2", + "evidence": { + "readOnlyHint": true + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "ambient", + "auth_type": null, + "credential_mode": null, + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "ambient", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='FileReadTool'].authority", + "evidence": { + "auth_type": null, + "credential_mode": null, + "scopes": [] + } + }, + { + "dimension": "authority", + "value": "ambient", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_authority", + "source_pointer": "/tools/2", + "evidence": { + "auth_type": null, + "scopes": [], + "alternative_count": 0 + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_97cdb0ffc03b3a364a1697b362308564d52d44b3dab46f799f1b7b5276adb212", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + } + }, + { + "tool_id": "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638", + "name": "summarize_case", + "provider": "crewai_agent", + "observation_ids": [ + "obs_v1_4da41a223c5b9819f534ad9f701d469aa605b258c479ae1768a5de3882e45d13", + "obs_v1_8107b85e09d544636b259f216489e077ba0c6915829b5e4a0c708d960d9b00df" + ], + "source_type": "crewai_inventory", + "source_ref": "inventories/tools.json", + "source_path": "inventories/tools.json", + "source_start_line": null, + "source_pointer": "/tools/1", + "risk_tags": [ + "read_only" + ], + "risk_tag_confidence": { + "read_only": "high" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "read", + "identity": { + "tool_id": "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638", + "status": "declared", + "provider": "crewai_agent", + "binding_id": "crewai_summarize_case", + "primary_observation_id": "obs_v1_4da41a223c5b9819f534ad9f701d469aa605b258c479ae1768a5de3882e45d13", + "observation_ids": [ + "obs_v1_4da41a223c5b9819f534ad9f701d469aa605b258c479ae1768a5de3882e45d13", + "obs_v1_8107b85e09d544636b259f216489e077ba0c6915829b5e4a0c708d960d9b00df" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_4da41a223c5b9819f534ad9f701d469aa605b258c479ae1768a5de3882e45d13", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "/tools/1", + "evidence": { + "source_type": "crewai_inventory", + "source_id": "crewai_inventory:inventories/tools.json", + "native_locator": "inventories/tools.json#summarize_case" + } + }, + { + "dimension": "identity", + "value": "obs_v1_8107b85e09d544636b259f216489e077ba0c6915829b5e4a0c708d960d9b00df", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "crew.py", + "evidence": { + "source_type": "crewai_function", + "source_id": "crewai_agent", + "native_locator": "crew.py#summarize_case" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='summarize_case'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/tools/1", + "evidence": { + "readOnlyHint": true + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "none", + "auth_type": null, + "credential_mode": null, + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='summarize_case'].authority", + "evidence": { + "auth_type": null, + "credential_mode": null, + "scopes": [] + } + }, + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_authority", + "source_pointer": "/tools/1", + "evidence": { + "auth_type": null, + "scopes": [], + "alternative_count": 0 + } + } + ], + "issues": [] + }, + "pass_eligible": true + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "reachable_path": [ + "agent_v1:db222e5ee84f5d3b4a7f0ca1", + "agent_v1:71a7997718e73610b0704464", + "tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:71a7997718e73610b0704464->tool_v2_d38c0379eb241f5063861aecb39b242b58e030f320ede64d7c862bfd6f68d638", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "crew.py", + "source_pointer": "crew.py:31", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true } } ], diff --git a/samples/simple_crewai_agent/expected/report.md b/samples/simple_crewai_agent/expected/report.md index f57053de..62d54e7e 100644 --- a/samples/simple_crewai_agent/expected/report.md +++ b/samples/simple_crewai_agent/expected/report.md @@ -14,7 +14,7 @@ Blockers (0): none Review items (1): - HIGH SHIP-AUTH-MISSING-SCOPE — FileReadTool lacks declared auth scopes -Evidence coverage: static (1 source warning(s); 1 semantic review concern(s); 2/3 actions pass-eligible; human review recommended) +Evidence coverage: static (1 source warning(s); 3/3 catalog tools reachable; 1 semantic review concern(s); 2/3 actions pass-eligible; human review recommended) Baseline delta: not enabled @@ -126,7 +126,14 @@ CrewAI warnings: - HIGH: SHIP-AUTH-MISSING-SCOPE [FileReadTool] - FileReadTool lacks declared auth scopes -## Appendix: Normalized Tool Inventory +## Agent Binding Surface + +Status: structural +Root agent: agent\_v1:db222e5ee84f5d3b4a7f0ca1 +Pass eligible: true +Catalog partition: 3 reachable, 0 possible, 0 unbound + +## Appendix: Root-Reachable Tool Inventory | Tool | Source | Risk Tags | Risk Confidence | Auth Scopes | Owner | | --- | --- | --- | --- | --- | --- | diff --git a/samples/simple_langchain_agent/expected/report.json b/samples/simple_langchain_agent/expected/report.json index 8b6de4a2..e94d6371 100644 --- a/samples/simple_langchain_agent/expected/report.json +++ b/samples/simple_langchain_agent/expected/report.json @@ -1,7 +1,7 @@ { "schema_version": "0.1", - "report_schema_version": "0.30", - "run_id": "agents_shipgate_28870fb1e06fe1de", + "report_schema_version": "0.31", + "run_id": "agents_shipgate_5051fb83bfbcadad", "manifest_dir": "/samples/simple_langchain_agent", "project": { "name": "simple-langchain-agent" @@ -85,6 +85,15 @@ "ambiguous_name_count": 0, "gap_count": 0, "reason_counts": {} + }, + "binding_coverage": { + "total_catalog_tools": 2, + "reachable_tools": 2, + "possible_tools": 0, + "unbound_tools": 0, + "pass_eligible": true, + "gap_count": 0, + "reason_counts": {} } }, "baseline_delta": { @@ -235,7 +244,7 @@ ] }, "action_surface_facts": { - "snapshot_version": "0.2", + "snapshot_version": "0.3", "actions": [ { "action_id": "agent:simple-langchain-agent/support-case-reader:action_v2_d0dd3e9ff4e2e56868595ef12e50a97b37c9619cad32007dd4155c3a8b5554a2", @@ -297,6 +306,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -455,6 +489,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -580,8 +639,60 @@ "modified": [], "notes": [ "No action-surface comparison source was provided." + ] + }, + "binding_surface_facts": { + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "status": "structural", + "agents": [ + { + "agent_id": "agent_v1:e72499e3feb23dae6e706766", + "name": "agent", + "source_id": "langchain_agent", + "source_ref": "agent.py", + "source_pointer": "agent.py:30" + } + ], + "tool_edges": [ + { + "agent_id": "agent_v1:e72499e3feb23dae6e706766", + "tool_id": "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "complete": true + }, + { + "agent_id": "agent_v1:e72499e3feb23dae6e706766", + "tool_id": "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "complete": true + } + ], + "handoff_edges": [], + "reachable_tool_ids": [ + "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c" ], - "tool_id": null + "possible_tool_ids": [], + "unbound_tool_ids": [], + "issues": [], + "pass_eligible": true + }, + "binding_surface_diff": { + "enabled": false, + "base_report_schema_version": null, + "added_reachable_tool_ids": [], + "removed_reachable_tool_ids": [], + "added_handoffs": [], + "removed_handoffs": [], + "notes": [] }, "capability_runtime_evidence": { "enabled": false, @@ -619,8 +730,8 @@ "findings": [], "recommended_actions": [], "generated_reports": { - "markdown": "expected/report.md", - "json": "expected/report.json" + "markdown": "/private/tmp/shipgate-golden-simple_langchain_agent/report.md", + "json": "/private/tmp/shipgate-golden-simple_langchain_agent/report.json" }, "loaded_policy_packs": [], "loaded_plugins": [], @@ -691,6 +802,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -755,6 +891,31 @@ "issues": [] }, "pass_eligible": true + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true } }, { @@ -822,6 +983,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -886,6 +1072,395 @@ "issues": [] }, "pass_eligible": true + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + } + } + ], + "tool_catalog": [ + { + "tool_id": "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "name": "summarize_case", + "provider": "langchain_agent", + "observation_ids": [ + "obs_v1_cca646e5e6b411bdae0262852ae9b5fc1a9cf8929ada5e68d65a3b8eec7db7c2", + "obs_v1_d044cf6a9ba34e66fc888092ff8c7f1ff38b73f4b2f0229fc23d290f6f92cf73" + ], + "source_type": "langchain_inventory", + "source_ref": "inventories/tools.json", + "source_path": "inventories/tools.json", + "source_start_line": null, + "source_pointer": "/tools/1", + "risk_tags": [ + "read_only" + ], + "risk_tag_confidence": { + "read_only": "high" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "read", + "identity": { + "tool_id": "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "status": "declared", + "provider": "langchain_agent", + "binding_id": "langchain_summarize_case", + "primary_observation_id": "obs_v1_cca646e5e6b411bdae0262852ae9b5fc1a9cf8929ada5e68d65a3b8eec7db7c2", + "observation_ids": [ + "obs_v1_cca646e5e6b411bdae0262852ae9b5fc1a9cf8929ada5e68d65a3b8eec7db7c2", + "obs_v1_d044cf6a9ba34e66fc888092ff8c7f1ff38b73f4b2f0229fc23d290f6f92cf73" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_cca646e5e6b411bdae0262852ae9b5fc1a9cf8929ada5e68d65a3b8eec7db7c2", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "/tools/1", + "evidence": { + "source_type": "langchain_inventory", + "source_id": "langchain_inventory:inventories/tools.json", + "native_locator": "inventories/tools.json#summarize_case" + } + }, + { + "dimension": "identity", + "value": "obs_v1_d044cf6a9ba34e66fc888092ff8c7f1ff38b73f4b2f0229fc23d290f6f92cf73", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "agent.py", + "evidence": { + "source_type": "langchain_structured_tool", + "source_id": "langchain_agent", + "native_locator": "agent.py#summarize_case" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='summarize_case'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/tools/1", + "evidence": { + "readOnlyHint": true + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "none", + "auth_type": null, + "credential_mode": null, + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='summarize_case'].authority", + "evidence": { + "auth_type": null, + "credential_mode": null, + "scopes": [] + } + }, + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_authority", + "source_pointer": "/tools/1", + "evidence": { + "auth_type": null, + "scopes": [], + "alternative_count": 0 + } + } + ], + "issues": [] + }, + "pass_eligible": true + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_2fdcf1b1fb2da2ef9bf3310f3998aa0942e03c878df2e00f7f99ba6437a2ce53", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + } + }, + { + "tool_id": "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c", + "name": "lookup_case", + "provider": "langchain_agent", + "observation_ids": [ + "obs_v1_5a5ea25e06f7f548a030228c69bb7fcc005eefc80b1c949897eb0a45f6633e74", + "obs_v1_6b9558cf467f2b12019c3d7494b38e8c31d8d1e7042b6a14318f01d1f7e76ab7" + ], + "source_type": "langchain_inventory", + "source_ref": "inventories/tools.json", + "source_path": "inventories/tools.json", + "source_start_line": null, + "source_pointer": "/tools/0", + "risk_tags": [ + "read_only" + ], + "risk_tag_confidence": { + "read_only": "high" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "read", + "identity": { + "tool_id": "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c", + "status": "declared", + "provider": "langchain_agent", + "binding_id": "langchain_lookup_case", + "primary_observation_id": "obs_v1_5a5ea25e06f7f548a030228c69bb7fcc005eefc80b1c949897eb0a45f6633e74", + "observation_ids": [ + "obs_v1_5a5ea25e06f7f548a030228c69bb7fcc005eefc80b1c949897eb0a45f6633e74", + "obs_v1_6b9558cf467f2b12019c3d7494b38e8c31d8d1e7042b6a14318f01d1f7e76ab7" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_5a5ea25e06f7f548a030228c69bb7fcc005eefc80b1c949897eb0a45f6633e74", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "/tools/0", + "evidence": { + "source_type": "langchain_inventory", + "source_id": "langchain_inventory:inventories/tools.json", + "native_locator": "inventories/tools.json#lookup_case" + } + }, + { + "dimension": "identity", + "value": "obs_v1_6b9558cf467f2b12019c3d7494b38e8c31d8d1e7042b6a14318f01d1f7e76ab7", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "agent.py", + "evidence": { + "source_type": "langchain_function", + "source_id": "langchain_agent", + "native_locator": "agent.py#lookup_case" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='lookup_case'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/tools/0", + "evidence": { + "readOnlyHint": true + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "none", + "auth_type": null, + "credential_mode": null, + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='lookup_case'].authority", + "evidence": { + "auth_type": null, + "credential_mode": null, + "scopes": [] + } + }, + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_authority", + "source_pointer": "/tools/0", + "evidence": { + "auth_type": null, + "scopes": [], + "alternative_count": 0 + } + } + ], + "issues": [] + }, + "pass_eligible": true + }, + "binding_assessment": { + "status": "structural", + "confidence": "high", + "root_agent_id": "agent_v1:e72499e3feb23dae6e706766", + "reachable_path": [ + "agent_v1:e72499e3feb23dae6e706766", + "tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:e72499e3feb23dae6e706766->tool_v2_d819fcc45add4884625d6fbdfacbb41d221868c520cb1c660e3f9ec5a6020a3c", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agent.py", + "source_pointer": "agent.py:30", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true } } ], diff --git a/samples/simple_langchain_agent/expected/report.md b/samples/simple_langchain_agent/expected/report.md index f66f53cb..3c62a8c8 100644 --- a/samples/simple_langchain_agent/expected/report.md +++ b/samples/simple_langchain_agent/expected/report.md @@ -13,7 +13,7 @@ Blockers (0): none Review items (0): none -Evidence coverage: static (2/2 actions pass-eligible) +Evidence coverage: static (2/2 catalog tools reachable; 2/2 actions pass-eligible) Baseline delta: not enabled @@ -91,7 +91,14 @@ No local runtime trace artifacts were declared for capability evidence. No findings. -## Appendix: Normalized Tool Inventory +## Agent Binding Surface + +Status: structural +Root agent: agent\_v1:e72499e3feb23dae6e706766 +Pass eligible: true +Catalog partition: 2 reachable, 0 possible, 0 unbound + +## Appendix: Root-Reachable Tool Inventory | Tool | Source | Risk Tags | Risk Confidence | Auth Scopes | Owner | | --- | --- | --- | --- | --- | --- | diff --git a/samples/simple_openai_api_agent/expected/report.json b/samples/simple_openai_api_agent/expected/report.json index 9735e309..bcb62405 100644 --- a/samples/simple_openai_api_agent/expected/report.json +++ b/samples/simple_openai_api_agent/expected/report.json @@ -1,7 +1,7 @@ { "schema_version": "0.1", - "report_schema_version": "0.30", - "run_id": "agents_shipgate_7717a23471c77185", + "report_schema_version": "0.31", + "run_id": "agents_shipgate_3189f29f2db532fe", "manifest_dir": "/samples/simple_openai_api_agent", "project": { "name": "simple-openai-api-agent", @@ -584,6 +584,15 @@ "ambiguous_name_count": 0, "gap_count": 0, "reason_counts": {} + }, + "binding_coverage": { + "total_catalog_tools": 2, + "reachable_tools": 2, + "possible_tools": 0, + "unbound_tools": 0, + "pass_eligible": true, + "gap_count": 0, + "reason_counts": {} } }, "baseline_delta": { @@ -813,6 +822,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -941,6 +975,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -1787,7 +1846,7 @@ ] }, "action_surface_facts": { - "snapshot_version": "0.2", + "snapshot_version": "0.3", "actions": [ { "action_id": "agent:simple-openai-api-agent/api-refund-assistant:action_v2_9d10be42545cd7457766e72c52ab9ba754eee7aa97c9613ae60ba4bc4048af82", @@ -1835,6 +1894,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -1994,6 +2078,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -2123,8 +2232,60 @@ "modified": [], "notes": [ "No action-surface comparison source was provided." + ] + }, + "binding_surface_facts": { + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "status": "declared", + "agents": [ + { + "agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "name": "root", + "source_id": null, + "source_ref": "shipgate.yaml", + "source_pointer": null + } ], - "tool_id": null + "tool_edges": [ + { + "agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "tool_id": "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "complete": true + }, + { + "agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "tool_id": "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "complete": true + } + ], + "handoff_edges": [], + "reachable_tool_ids": [ + "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865" + ], + "possible_tool_ids": [], + "unbound_tool_ids": [], + "issues": [], + "pass_eligible": true + }, + "binding_surface_diff": { + "enabled": false, + "base_report_schema_version": null, + "added_reachable_tool_ids": [], + "removed_reachable_tool_ids": [], + "added_handoffs": [], + "removed_handoffs": [], + "notes": [] }, "capability_runtime_evidence": { "enabled": true, @@ -3122,8 +3283,8 @@ "Declare operation-specific auth scopes for send_customer_email, or explicitly declare anonymous authority when the operation requires no credentials." ], "generated_reports": { - "markdown": "expected/report.md", - "json": "expected/report.json" + "markdown": "/private/tmp/shipgate-golden-simple_openai_api_agent/report.md", + "json": "/private/tmp/shipgate-golden-simple_openai_api_agent/report.json" }, "loaded_policy_packs": [], "loaded_plugins": [], @@ -3183,6 +3344,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -3259,6 +3445,31 @@ "issues": [] }, "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true } }, { @@ -3313,6 +3524,31 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, "effect": { "status": "declared", "confidence": "high", @@ -3378,6 +3614,384 @@ "issues": [] }, "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + } + } + ], + "tool_catalog": [ + { + "tool_id": "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "name": "send_customer_email", + "provider": "openai_api", + "observation_ids": [ + "obs_v1_6975b831c4f5ddcc793bb748ab778dfb271e6372197d26849a3c6a45ac5aefe8" + ], + "source_type": "openai_api", + "source_ref": "tools/openai-tools.json#1", + "source_path": "tools/openai-tools.json", + "source_start_line": null, + "source_pointer": "/tools/1", + "risk_tags": [ + "customer_communication", + "external_write", + "write" + ], + "risk_tag_confidence": { + "customer_communication": "high", + "external_write": "high", + "write": "medium" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "external_communication", + "identity": { + "tool_id": "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "status": "structural", + "provider": "openai_api", + "binding_id": null, + "primary_observation_id": "obs_v1_6975b831c4f5ddcc793bb748ab778dfb271e6372197d26849a3c6a45ac5aefe8", + "observation_ids": [ + "obs_v1_6975b831c4f5ddcc793bb748ab778dfb271e6372197d26849a3c6a45ac5aefe8" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_6975b831c4f5ddcc793bb748ab778dfb271e6372197d26849a3c6a45ac5aefe8", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "source_observation", + "source_pointer": "/tools/1", + "evidence": { + "source_type": "openai_api", + "source_id": "openai_api", + "native_locator": "tools/openai-tools.json#1#send_customer_email" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='send_customer_email'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/tools/1", + "evidence": { + "tag": "customer_communication", + "hint_source": "keyword" + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/tools/1", + "evidence": { + "tag": "external_write", + "hint_source": "keyword" + } + }, + { + "dimension": "effect", + "value": "write", + "confidence": "medium", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:openai_api_keyword", + "source_pointer": "/tools/1", + "evidence": { + "tag": "write", + "hint_source": "openai_api_keyword" + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "unscoped", + "auth_type": "api_key", + "credential_mode": "service_account", + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "unscoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='send_customer_email'].authority", + "evidence": { + "auth_type": "api_key", + "credential_mode": "service_account", + "scopes": [] + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_99ba7c14d528f47da6093918bb937aafc5d75dd4f4972797c8e9697c14bcb61b", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + } + }, + { + "tool_id": "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "name": "create_refund", + "provider": "openai_api", + "observation_ids": [ + "obs_v1_650c1e14249a3ef5447890a74dbb1d7b0c71682d4a1c61a888343a6381d6a10e" + ], + "source_type": "openai_api", + "source_ref": "tools/openai-tools.json#0", + "source_path": "tools/openai-tools.json", + "source_start_line": null, + "source_pointer": "/tools/0", + "risk_tags": [ + "financial_action", + "write" + ], + "risk_tag_confidence": { + "financial_action": "high", + "write": "medium" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "financial_write", + "identity": { + "tool_id": "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "status": "structural", + "provider": "openai_api", + "binding_id": null, + "primary_observation_id": "obs_v1_650c1e14249a3ef5447890a74dbb1d7b0c71682d4a1c61a888343a6381d6a10e", + "observation_ids": [ + "obs_v1_650c1e14249a3ef5447890a74dbb1d7b0c71682d4a1c61a888343a6381d6a10e" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_650c1e14249a3ef5447890a74dbb1d7b0c71682d4a1c61a888343a6381d6a10e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "source_observation", + "source_pointer": "/tools/0", + "evidence": { + "source_type": "openai_api", + "source_id": "openai_api", + "native_locator": "tools/openai-tools.json#0#create_refund" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "financial_write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='create_refund'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "financial_write", + "confidence": "high", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/tools/0", + "evidence": { + "tag": "financial_action", + "hint_source": "keyword", + "scopes": [] + } + }, + { + "dimension": "effect", + "value": "write", + "confidence": "medium", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:openai_api_keyword", + "source_pointer": "/tools/0", + "evidence": { + "tag": "write", + "hint_source": "openai_api_keyword" + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "unscoped", + "auth_type": "api_key", + "credential_mode": "service_account", + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "unscoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='create_refund'].authority", + "evidence": { + "auth_type": "api_key", + "credential_mode": "service_account", + "scopes": [] + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:3e1354866b28ba54f69c8e73", + "reachable_path": [ + "agent_v1:3e1354866b28ba54f69c8e73", + "tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:3e1354866b28ba54f69c8e73->tool_v2_b900644b842a2ec3ce8827653cf7153ac2b7f03e18f92566f5c12058bb368865", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [], + "pass_eligible": true } } ], diff --git a/samples/simple_openai_api_agent/expected/report.md b/samples/simple_openai_api_agent/expected/report.md index c858ff56..8b373786 100644 --- a/samples/simple_openai_api_agent/expected/report.md +++ b/samples/simple_openai_api_agent/expected/report.md @@ -34,7 +34,7 @@ Review items (19): - HIGH SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING — send\_customer\_email is high-risk but has no owner - HIGH SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING — create\_refund is high-risk but has no owner -Evidence coverage: static (2 semantic review concern(s); 0/2 actions pass-eligible; human review recommended) +Evidence coverage: static (2/2 catalog tools reachable; 2 semantic review concern(s); 0/2 actions pass-eligible; human review recommended) Baseline delta: not enabled @@ -229,7 +229,14 @@ Matched trace rows: - HIGH: SHIP-SIDEFX-IDEMPOTENCY-MISSING [create\_refund] - create\_refund lacks idempotency evidence -## Appendix: Normalized Tool Inventory +## Agent Binding Surface + +Status: declared +Root agent: agent\_v1:3e1354866b28ba54f69c8e73 +Pass eligible: true +Catalog partition: 2 reachable, 0 possible, 0 unbound + +## Appendix: Root-Reachable Tool Inventory | Tool | Source | Risk Tags | Risk Confidence | Auth Scopes | Owner | | --- | --- | --- | --- | --- | --- | diff --git a/samples/simple_openai_api_agent/shipgate.yaml b/samples/simple_openai_api_agent/shipgate.yaml index 2afde698..37e2af4a 100644 --- a/samples/simple_openai_api_agent/shipgate.yaml +++ b/samples/simple_openai_api_agent/shipgate.yaml @@ -38,6 +38,16 @@ openai_api: policy_rules: - path: policies/openai-api-policy.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: create_refund} + - {tool: send_customer_email} + handoffs: [] + reason: reviewed OpenAI API fixture binding + action_surface: actions: - tool: create_refund diff --git a/samples/support_refund_agent/expected/packet.html b/samples/support_refund_agent/expected/packet.html index 5427dfdc..ebe6783a 100644 --- a/samples/support_refund_agent/expected/packet.html +++ b/samples/support_refund_agent/expected/packet.html @@ -26,4 +26,4 @@ .status-missing { color: #7f1d1d; } .status-informational { color: #555; } .meta { color: #555; font-size: 0.92rem; } -

Release Evidence Packet

Project: support-refund-agent · Agent: refund-assistant · Environment: production_like
Run id: agents_shipgate_7429a94499efb630 · Generated at: 2026-01-01T00:00:00+00:00 · Packet schema: 0.9

This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See §10 for what the packet does not prove.

§1 Release decision — BLOCKED

  • Decision: blocked
  • Reason: 5 active findings block release.
  • Blockers: 5
  • Review items: 15

CI gate behavior (informational)

  • ci_mode: advisory, would_fail_ci: false, exit code: 0
  • Note: CI behavior is metadata about the run gate, not the verdict. The verdict above derives from release_decision.decision.

Static semantic coverage

  • Pass-eligible actions: 6/8
  • Evidence gaps: 3
  • Known authority review concerns: 1
  • Reasons: incomplete_surface=1, missing_authority_evidence=1, missing_effect_evidence=1, unscoped_authority=1

Blockers

  • SHIP-POLICY-APPROVAL-MISSING (critical): stripe.create_refund lacks a declared approval policy
  • SHIP-SIDEFX-IDEMPOTENCY-MISSING (critical): stripe.create_refund lacks idempotency evidence
  • SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING (high): gmail.send_customer_email has external communication capability without required controls
  • SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING (high): stripe.create_refund has external communication capability without required controls
  • SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING (critical): stripe.create_refund has financial write capability without required controls

Review items

  • SHIP-INVENTORY-WILDCARD-TOOLS (high): Wildcard tool exposure declared
  • SHIP-SCHEMA-MISSING-BOUNDS (high): stripe.create_refund.amount has no maximum bound
  • SHIP-SCHEMA-BROAD-FREE-TEXT (high): zendesk.update_ticket accepts broad free-form action input
  • SHIP-SCHEMA-BROAD-FREE-TEXT (high): gmail.send_customer_email accepts broad free-form action input
  • SHIP-AUTH-MANIFEST-BROAD-SCOPE (high): Manifest declares broad permission scopes
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): shopify.cancel_order requires scopes not declared in the manifest
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): support.search_kb requires scopes not declared in the manifest
  • SHIP-AUTH-MISSING-SCOPE (high): refund_status_lookup lacks declared auth scopes
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): gmail.send_customer_email requires scopes not declared in the manifest
  • SHIP-SCOPE-PROHIBITED-TOOL-PRESENT (high): stripe.create_refund appears to overlap with a prohibited action
  • SHIP-SCOPE-PROHIBITED-TOOL-PRESENT (high): gmail.send_customer_email appears to overlap with a prohibited action
  • SHIP-POLICY-CONFIRMATION-MISSING (high): stripe.create_refund lacks a declared confirmation policy
  • SHIP-POLICY-CONFIRMATION-MISSING (high): gmail.send_customer_email lacks a declared confirmation policy
  • SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING (high): shopify.cancel_order is high-risk but has no owner
  • SHIP-MANIFEST-UNUSED-SCOPE (medium): Manifest declares unused permission scope zendesk:tickets:read

§1A Evidence matrix — compact review summary

  • Evidence Matrix Light is derived from public report.json only. Release decisions, CI exit behavior, and baseline semantics remain owned by release_decision. Domain rows intentionally overlap; a single finding can appear in multiple rows when it is relevant to each review lens.
DomainEvidence presentEvidence sourceConfidenceMissing controlsBlocking findingsReview items
Inventorypartialtool_inventory; tool_surface; +2 morehighSHIP-INVENTORY-WILDCARD-TOOLS on wildcard_mcp_tools.*: Wildcard tool exposure declared—SHIP-INVENTORY-WILDCARD-TOOLS (high)
Schemapartialtool_surface_facts.tools[].hashes; findings[]mixedSHIP-SCHEMA-MISSING-BOUNDS on stripe.create_refund: stripe.create_refund.amount has no maximum bound; SHIP-SCHEMA-BROAD-FREE-TEXT on zendesk.update_ticket: zendesk.update_ticket accepts broad free-form action input; +1 more—SHIP-SCHEMA-MISSING-BOUNDS (high); SHIP-SCHEMA-BROAD-FREE-TEXT (high); +1 more
Authpartialtool_surface_facts.scopes; tool_inventory[].auth_scopes; +1 moremixedSHIP-AUTH-MANIFEST-BROAD-SCOPE: Manifest declares broad permission scopes; SHIP-AUTH-SCOPE-COVERAGE-MISSING on shopify.cancel_order: shopify.cancel_order requires scopes not declared in the manifest; +4 more—SHIP-AUTH-MANIFEST-BROAD-SCOPE (high); SHIP-AUTH-SCOPE-COVERAGE-MISSING (high); +4 more
Approvalpartialtool_surface_facts.controls[kind=approval_policy]; findings[]highSHIP-POLICY-APPROVAL-MISSING on stripe.create_refund: stripe.create_refund lacks a declared approval policy; SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING on stripe.create_refund: stripe.create_refund has financial write capability without required controlsSHIP-POLICY-APPROVAL-MISSING (critical); SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING (critical)—
Confirmationpartialtool_surface_facts.controls[kind=confirmation_policy]; findings[]highSHIP-POLICY-CONFIRMATION-MISSING on stripe.create_refund: stripe.create_refund lacks a declared confirmation policy; SHIP-POLICY-CONFIRMATION-MISSING on gmail.send_customer_email: gmail.send_customer_email lacks a declared confirmation policy—SHIP-POLICY-CONFIRMATION-MISSING (high); SHIP-POLICY-CONFIRMATION-MISSING (high)
Idempotencypartialtool_surface_facts.controls[kind=idempotency_evidence]; action_surface_facts.actions[].safeguards.idempotency; +1 morehighSHIP-SIDEFX-IDEMPOTENCY-MISSING on stripe.create_refund: stripe.create_refund lacks idempotency evidence; SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING on stripe.create_refund: stripe.create_refund has financial write capability without required controlsSHIP-SIDEFX-IDEMPOTENCY-MISSING (critical); SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING (critical)—
Side effectspartialtool_inventory[].risk_tags; action_surface_facts.actions[].effect; +1 moremixedSHIP-SCHEMA-BROAD-FREE-TEXT on zendesk.update_ticket: zendesk.update_ticket accepts broad free-form action input; SHIP-SCHEMA-BROAD-FREE-TEXT on gmail.send_customer_email: gmail.send_customer_email accepts broad free-form action input; +7 moreSHIP-POLICY-APPROVAL-MISSING (critical); SHIP-SIDEFX-IDEMPOTENCY-MISSING (critical); +3 moreSHIP-SCHEMA-BROAD-FREE-TEXT (high); SHIP-SCHEMA-BROAD-FREE-TEXT (high); +2 more
Memory isolationnot_declared—unknown———
Human-in-the-loop evidencenot_declared—unknown———
Prompt/scope alignmentpartialdeclared_intentions; misalignments; +2 moremediumSHIP-SCOPE-PROHIBITED-TOOL-PRESENT on stripe.create_refund: stripe.create_refund appears to overlap with a prohibited action; SHIP-SCOPE-PROHIBITED-TOOL-PRESENT on gmail.send_customer_email: gmail.send_customer_email appears to overlap with a prohibited action—SHIP-SCOPE-PROHIBITED-TOOL-PRESENT (high); SHIP-SCOPE-PROHIBITED-TOOL-PRESENT (high)
Retry/timeoutnot_declared—unknown———
Baseline debtinformational—unknown———
Action-surface policypartialaction_surface_facts.actions; findings[].blocks_release; +1 morehighSHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING on gmail.send_customer_email: gmail.send_customer_email has external communication capability without required controls; SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING on stripe.create_refund: stripe.create_refund has external communication capability without required controls; +1 moreSHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING (high); SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING (high); +1 more—

§2 Capability ↔ Intent diff — missing

Declared

  • Purpose: answer refund policy questions
  • Purpose: prepare refund requests for human review
  • Purpose: update support ticket notes
  • Prohibited: issue refund without approval
  • Prohibited: cancel order without explicit confirmation
  • Prohibited: send external email without preview

Observed tools

  • gmail.send_customer_email
  • refund_status_lookup
  • send_email_preview
  • shopify.cancel_order
  • stripe.create_refund
  • support.search_kb
  • wildcard_mcp_tools.*
  • zendesk.update_ticket

Divergences

  • SHIP-SCOPE-PROHIBITED-TOOL-PRESENT: stripe.create_refund appears to overlap with a prohibited action
  • SHIP-SCOPE-PROHIBITED-TOOL-PRESENT: gmail.send_customer_email appears to overlap with a prohibited action

§3 High-risk tool surface — partial

Total tools: 8 · High-risk: 3

ToolSourceRisk tagsApprovalIdempotency
gmail.send_customer_emailmcpcustomer_communication, external_writenono
shopify.cancel_orderopenapidestructive, writeyesyes
stripe.create_refundopenapiexternal_write, financial_action, writenono

§3A Tool-surface diff — not declared

Status: disabled — No --diff-from report or v0.3 baseline snapshot was provided.
Base: none

§3B Action-surface diff — not declared

Status: disabled — No action-surface comparison source was provided.
Base: none

§4 Approval policy coverage — partial

ToolDeclaredSourceGap finding(s)
shopify.cancel_orderyespolicies—
stripe.create_refundno—fp_973ea0ef2110ca9a

Gap findings

  • SHIP-POLICY-APPROVAL-MISSING (critical): stripe.create_refund lacks a declared approval policy

§5 Idempotency / retry risk — partial

Retry policy: not declared

ToolDeclaredSourceGap finding(s)
shopify.cancel_orderyespolicies—
stripe.create_refundno—fp_2cf0d6c77d9c3eee

Gap findings

  • SHIP-SIDEFX-IDEMPOTENCY-MISSING (critical): stripe.create_refund lacks idempotency evidence

§6 Scope coverage — missing

Declared scopes

  • zendesk:tickets:read
  • zendesk:tickets:write
  • stripe:*
ScopeDeclaredUsed by tools
gmail:sendnogmail.send_customer_email
shopify:orders:writenoshopify.cancel_order
stripe:*yes—
stripe:refunds:writeyesstripe.create_refund
support:kb:readnosupport.search_kb
zendesk:tickets:readyes—
zendesk:tickets:writeyeszendesk.update_ticket

Unused declared scopes

  • zendesk:tickets:read

Used by tools but not declared

  • gmail:send
  • shopify:orders:write
  • support:kb:read

Gap findings

  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): shopify.cancel_order requires scopes not declared in the manifest
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): support.search_kb requires scopes not declared in the manifest
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): gmail.send_customer_email requires scopes not declared in the manifest
  • SHIP-MANIFEST-UNUSED-SCOPE (medium): Manifest declares unused permission scope zendesk:tickets:read

§7 Memory isolation — not declared

Manifest does not declare a memory isolation policy. The current manifest schema (v0.1) has no agent.memory field. See §10 for the residual review item.

§8 Human-in-the-loop evidence — covered

  • Configured: yes
  • Human review recommended: yes
  • Provenance mode: fresh_scan
  • Capability-linked trace rows: 0/0 matched (0 source(s))
  • HITL evidence is local review evidence only. Missing local evidence does not prove a runtime control is absent, and present local evidence does not certify runtime enforcement.

Approval-required tools

  • shopify.cancel_order

Confirmation-required tools

  • shopify.cancel_order

§9 Required dynamic scenarios — partial

  • Manual review for SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING — Declare confirmation policy and safeguards.audit_log for this external communication action.
    Related finding(s): fp_1c94d2d2693dccdf, fp_e042ce7813b97a2d
  • Manual review for SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING — Declare approval.required, safeguards.audit_log, and safeguards.idempotency for this financial write action.
    Related finding(s): fp_dfa27ad5b52d8fd6
  • Manual review for SHIP-AUTH-MANIFEST-BROAD-SCOPE — Replace broad manifest permission scopes with the narrowest scopes needed for this release.
    Related finding(s): fp_df4a990cca9f936b
  • Manual review for SHIP-AUTH-MISSING-SCOPE — Declare operation-specific auth scopes for refund_status_lookup, or explicitly declare anonymous authority when the operation requires no credentials.
    Related finding(s): fp_519cb82f038efd10
  • Manual review for SHIP-AUTH-SCOPE-COVERAGE-MISSING — Add the required scopes for shopify.cancel_order to permissions.scopes or narrow the tool's declared auth requirements.
    Related finding(s): fp_095f3a5337124f6e, fp_1fd01b4ed2e41d51, fp_24d610b0d4324190
  • Manual review for SHIP-INVENTORY-WILDCARD-TOOLS — Replace wildcard tool exposure with an explicit tool allowlist before release review.
    Related finding(s): fp_a00e34cdb33a5596
  • Manual review for SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING — Declare an owner for each high-risk production tool in risk_overrides.tools.
    Related finding(s): fp_674fa79ae9993422
  • Manual review for SHIP-MANIFEST-UNUSED-SCOPE — Remove unused manifest scopes or add tool metadata showing why they are required.
    Related finding(s): fp_609d62f4dc434961
  • Manual review for SHIP-POLICY-APPROVAL-MISSING — Declare an approval policy for stripe.create_refund or remove this tool from the release.
    Related finding(s): fp_973ea0ef2110ca9a
  • Manual review for SHIP-POLICY-CONFIRMATION-MISSING — Declare a user confirmation policy for stripe.create_refund or remove this action from the release.
    Related finding(s): fp_c762eebfadaf39d9, fp_fae2921fd2d0cbd5
  • Manual review for SHIP-SCHEMA-BROAD-FREE-TEXT — Constrain zendesk.update_ticket.updates with an enum, structured schema, or narrower field-specific parameters.
    Related finding(s): fp_d58657d9b1c91a84, fp_ea4f18e4707f8e1d
  • Manual review for SHIP-SCHEMA-MISSING-BOUNDS — Add a maximum bound to stripe.create_refund.amount or document an equivalent limit in the tool policy.
    Related finding(s): fp_60fdf92126ba8844
  • Manual review for SHIP-SCOPE-PROHIBITED-TOOL-PRESENT — Remove stripe.create_refund, narrow its policy, or revise prohibited_actions so the manifest and tool surface do not contradict each other.
    Related finding(s): fp_5faaaee860bcec8b, fp_6f8e1ebab65f3607
  • Manual review for SHIP-SIDEFX-IDEMPOTENCY-MISSING — Add an idempotency key, idempotent annotation, or declared idempotency policy for stripe.create_refund.
    Related finding(s): fp_2cf0d6c77d9c3eee
  • Re-run scan after resolving source warnings — Source loaders emitted warnings; some tool surfaces may have been parsed with reduced confidence.

§10 What this packet did NOT prove

Agents Shipgate is an advisory tool: the deterministic merge gate for AI-generated agent capability changes, run as a local-first, static Tool-Use Readiness review. The packet below is derived from a scan; it does not, by itself, prove the following properties:

  • Prompt robustness. Whether the agent's prompt holds up under jailbreaks, persona drift, indirect prompt injection, or adversarial inputs.
  • Runtime behavior. Whether the agent actually invokes only the declared tools, respects approval gates at runtime, or follows policy under load. Static config is not runtime evidence.
  • Model correctness. Whether the underlying model produces correct outputs, calls the right tools, or stays within the declared scope. The packet does not benchmark the model.
  • Adversarial resistance. Whether the agent withstands red-team or penetration testing. The packet does not run scenarios; it organizes evidence.

Per-run residuals

  • Source warnings:
    • MCP source declares wildcard tool exposure
  • Low-confidence tool extractions: none
  • Suppressed findings in effect: none
  • Memory isolation is not modeled by the v0.1 manifest schema; no static evidence is available.
  • 5 active finding(s) came from heuristic provenance (keyword_heuristic=5, regex_heuristic=0); review the finding evidence before acting.
+

Release Evidence Packet

Project: support-refund-agent · Agent: refund-assistant · Environment: production_like
Run id: agents_shipgate_5ad1a92a984aacac · Generated at: 2026-01-01T00:00:00+00:00 · Packet schema: 0.10

This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See §10 for what the packet does not prove.

§1 Release decision — BLOCKED

  • Decision: blocked
  • Reason: 5 active findings block release.
  • Blockers: 5
  • Review items: 14

CI gate behavior (informational)

  • ci_mode: advisory, would_fail_ci: false, exit code: 0
  • Note: CI behavior is metadata about the run gate, not the verdict. The verdict above derives from release_decision.decision.

Static semantic coverage

  • Pass-eligible actions: 0/7
  • Evidence gaps: 7
  • Known authority review concerns: 1
  • Reasons: conflicting_binding_evidence=7, unscoped_authority=1

Blockers

  • SHIP-POLICY-APPROVAL-MISSING (critical): stripe.create_refund lacks a declared approval policy
  • SHIP-SIDEFX-IDEMPOTENCY-MISSING (critical): stripe.create_refund lacks idempotency evidence
  • SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING (high): gmail.send_customer_email has external communication capability without required controls
  • SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING (high): stripe.create_refund has external communication capability without required controls
  • SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING (critical): stripe.create_refund has financial write capability without required controls

Review items

  • SHIP-SCHEMA-MISSING-BOUNDS (high): stripe.create_refund.amount has no maximum bound
  • SHIP-SCHEMA-BROAD-FREE-TEXT (high): zendesk.update_ticket accepts broad free-form action input
  • SHIP-SCHEMA-BROAD-FREE-TEXT (high): gmail.send_customer_email accepts broad free-form action input
  • SHIP-AUTH-MANIFEST-BROAD-SCOPE (high): Manifest declares broad permission scopes
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): shopify.cancel_order requires scopes not declared in the manifest
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): support.search_kb requires scopes not declared in the manifest
  • SHIP-AUTH-MISSING-SCOPE (high): refund_status_lookup lacks declared auth scopes
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): gmail.send_customer_email requires scopes not declared in the manifest
  • SHIP-SCOPE-PROHIBITED-TOOL-PRESENT (high): stripe.create_refund appears to overlap with a prohibited action
  • SHIP-SCOPE-PROHIBITED-TOOL-PRESENT (high): gmail.send_customer_email appears to overlap with a prohibited action
  • SHIP-POLICY-CONFIRMATION-MISSING (high): stripe.create_refund lacks a declared confirmation policy
  • SHIP-POLICY-CONFIRMATION-MISSING (high): gmail.send_customer_email lacks a declared confirmation policy
  • SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING (high): shopify.cancel_order is high-risk but has no owner
  • SHIP-MANIFEST-UNUSED-SCOPE (medium): Manifest declares unused permission scope zendesk:tickets:read

§1A Evidence matrix — compact review summary

  • Evidence Matrix Light is derived from public report.json only. Release decisions, CI exit behavior, and baseline semantics remain owned by release_decision. Domain rows intentionally overlap; a single finding can appear in multiple rows when it is relevant to each review lens.
DomainEvidence presentEvidence sourceConfidenceMissing controlsBlocking findingsReview items
Inventorycoveredtool_inventory; tool_surface; +1 moremedium———
Schemapartialtool_surface_facts.tools[].hashes; findings[]mixedSHIP-SCHEMA-MISSING-BOUNDS on stripe.create_refund: stripe.create_refund.amount has no maximum bound; SHIP-SCHEMA-BROAD-FREE-TEXT on zendesk.update_ticket: zendesk.update_ticket accepts broad free-form action input; +1 more—SHIP-SCHEMA-MISSING-BOUNDS (high); SHIP-SCHEMA-BROAD-FREE-TEXT (high); +1 more
Authpartialtool_surface_facts.scopes; tool_inventory[].auth_scopes; +1 moremixedSHIP-AUTH-MANIFEST-BROAD-SCOPE: Manifest declares broad permission scopes; SHIP-AUTH-SCOPE-COVERAGE-MISSING on shopify.cancel_order: shopify.cancel_order requires scopes not declared in the manifest; +4 more—SHIP-AUTH-MANIFEST-BROAD-SCOPE (high); SHIP-AUTH-SCOPE-COVERAGE-MISSING (high); +4 more
Approvalpartialtool_surface_facts.controls[kind=approval_policy]; findings[]highSHIP-POLICY-APPROVAL-MISSING on stripe.create_refund: stripe.create_refund lacks a declared approval policy; SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING on stripe.create_refund: stripe.create_refund has financial write capability without required controlsSHIP-POLICY-APPROVAL-MISSING (critical); SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING (critical)—
Confirmationpartialtool_surface_facts.controls[kind=confirmation_policy]; findings[]highSHIP-POLICY-CONFIRMATION-MISSING on stripe.create_refund: stripe.create_refund lacks a declared confirmation policy; SHIP-POLICY-CONFIRMATION-MISSING on gmail.send_customer_email: gmail.send_customer_email lacks a declared confirmation policy—SHIP-POLICY-CONFIRMATION-MISSING (high); SHIP-POLICY-CONFIRMATION-MISSING (high)
Idempotencypartialtool_surface_facts.controls[kind=idempotency_evidence]; action_surface_facts.actions[].safeguards.idempotency; +1 morehighSHIP-SIDEFX-IDEMPOTENCY-MISSING on stripe.create_refund: stripe.create_refund lacks idempotency evidence; SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING on stripe.create_refund: stripe.create_refund has financial write capability without required controlsSHIP-SIDEFX-IDEMPOTENCY-MISSING (critical); SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING (critical)—
Side effectspartialtool_inventory[].risk_tags; action_surface_facts.actions[].effect; +1 moremixedSHIP-SCHEMA-BROAD-FREE-TEXT on zendesk.update_ticket: zendesk.update_ticket accepts broad free-form action input; SHIP-SCHEMA-BROAD-FREE-TEXT on gmail.send_customer_email: gmail.send_customer_email accepts broad free-form action input; +7 moreSHIP-POLICY-APPROVAL-MISSING (critical); SHIP-SIDEFX-IDEMPOTENCY-MISSING (critical); +3 moreSHIP-SCHEMA-BROAD-FREE-TEXT (high); SHIP-SCHEMA-BROAD-FREE-TEXT (high); +2 more
Memory isolationnot_declared—unknown———
Human-in-the-loop evidencenot_declared—unknown———
Prompt/scope alignmentpartialdeclared_intentions; misalignments; +2 moremediumSHIP-SCOPE-PROHIBITED-TOOL-PRESENT on stripe.create_refund: stripe.create_refund appears to overlap with a prohibited action; SHIP-SCOPE-PROHIBITED-TOOL-PRESENT on gmail.send_customer_email: gmail.send_customer_email appears to overlap with a prohibited action—SHIP-SCOPE-PROHIBITED-TOOL-PRESENT (high); SHIP-SCOPE-PROHIBITED-TOOL-PRESENT (high)
Retry/timeoutnot_declared—unknown———
Baseline debtinformational—unknown———
Action-surface policypartialaction_surface_facts.actions; findings[].blocks_release; +1 morehighSHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING on gmail.send_customer_email: gmail.send_customer_email has external communication capability without required controls; SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING on stripe.create_refund: stripe.create_refund has external communication capability without required controls; +1 moreSHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING (high); SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING (high); +1 more—

§2 Capability ↔ Intent diff — missing

Declared

  • Purpose: answer refund policy questions
  • Purpose: prepare refund requests for human review
  • Purpose: update support ticket notes
  • Prohibited: issue refund without approval
  • Prohibited: cancel order without explicit confirmation
  • Prohibited: send external email without preview

Observed tools

  • gmail.send_customer_email
  • refund_status_lookup
  • send_email_preview
  • shopify.cancel_order
  • stripe.create_refund
  • support.search_kb
  • zendesk.update_ticket

Divergences

  • SHIP-SCOPE-PROHIBITED-TOOL-PRESENT: stripe.create_refund appears to overlap with a prohibited action
  • SHIP-SCOPE-PROHIBITED-TOOL-PRESENT: gmail.send_customer_email appears to overlap with a prohibited action

§3 High-risk tool surface — partial

Total tools: 7 · High-risk: 3

ToolSourceRisk tagsApprovalIdempotency
gmail.send_customer_emailmcpcustomer_communication, external_writenono
shopify.cancel_orderopenapidestructive, writeyesyes
stripe.create_refundopenapiexternal_write, financial_action, writenono

§3A Tool-surface diff — not declared

Status: disabled — No --diff-from report or v0.3 baseline snapshot was provided.
Base: none

§3B Action-surface diff — not declared

Status: disabled — No action-surface comparison source was provided.
Base: none

§4 Approval policy coverage — partial

ToolDeclaredSourceGap finding(s)
shopify.cancel_orderyespolicies—
stripe.create_refundno—fp_973ea0ef2110ca9a

Gap findings

  • SHIP-POLICY-APPROVAL-MISSING (critical): stripe.create_refund lacks a declared approval policy

§5 Idempotency / retry risk — partial

Retry policy: not declared

ToolDeclaredSourceGap finding(s)
shopify.cancel_orderyespolicies—
stripe.create_refundno—fp_2cf0d6c77d9c3eee

Gap findings

  • SHIP-SIDEFX-IDEMPOTENCY-MISSING (critical): stripe.create_refund lacks idempotency evidence

§6 Scope coverage — missing

Declared scopes

  • zendesk:tickets:read
  • zendesk:tickets:write
  • stripe:*
ScopeDeclaredUsed by tools
gmail:sendnogmail.send_customer_email
shopify:orders:writenoshopify.cancel_order
stripe:*yes—
stripe:refunds:writeyesstripe.create_refund
support:kb:readnosupport.search_kb
zendesk:tickets:readyes—
zendesk:tickets:writeyeszendesk.update_ticket

Unused declared scopes

  • zendesk:tickets:read

Used by tools but not declared

  • gmail:send
  • shopify:orders:write
  • support:kb:read

Gap findings

  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): shopify.cancel_order requires scopes not declared in the manifest
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): support.search_kb requires scopes not declared in the manifest
  • SHIP-AUTH-SCOPE-COVERAGE-MISSING (high): gmail.send_customer_email requires scopes not declared in the manifest
  • SHIP-MANIFEST-UNUSED-SCOPE (medium): Manifest declares unused permission scope zendesk:tickets:read

§7 Memory isolation — not declared

Manifest does not declare a memory isolation policy. The current manifest schema (v0.1) has no agent.memory field. See §10 for the residual review item.

§8 Human-in-the-loop evidence — covered

  • Configured: yes
  • Human review recommended: yes
  • Provenance mode: fresh_scan
  • Capability-linked trace rows: 0/0 matched (0 source(s))
  • HITL evidence is local review evidence only. Missing local evidence does not prove a runtime control is absent, and present local evidence does not certify runtime enforcement.

Approval-required tools

  • shopify.cancel_order

Confirmation-required tools

  • shopify.cancel_order

§9 Required dynamic scenarios — partial

  • Manual review for SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING — Declare confirmation policy and safeguards.audit_log for this external communication action.
    Related finding(s): fp_1c94d2d2693dccdf, fp_e042ce7813b97a2d
  • Manual review for SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING — Declare approval.required, safeguards.audit_log, and safeguards.idempotency for this financial write action.
    Related finding(s): fp_dfa27ad5b52d8fd6
  • Manual review for SHIP-AUTH-MANIFEST-BROAD-SCOPE — Replace broad manifest permission scopes with the narrowest scopes needed for this release.
    Related finding(s): fp_df4a990cca9f936b
  • Manual review for SHIP-AUTH-MISSING-SCOPE — Declare operation-specific auth scopes for refund_status_lookup, or explicitly declare anonymous authority when the operation requires no credentials.
    Related finding(s): fp_519cb82f038efd10
  • Manual review for SHIP-AUTH-SCOPE-COVERAGE-MISSING — Add the required scopes for shopify.cancel_order to permissions.scopes or narrow the tool's declared auth requirements.
    Related finding(s): fp_095f3a5337124f6e, fp_1fd01b4ed2e41d51, fp_24d610b0d4324190
  • Manual review for SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING — Declare an owner for each high-risk production tool in risk_overrides.tools.
    Related finding(s): fp_674fa79ae9993422
  • Manual review for SHIP-MANIFEST-UNUSED-SCOPE — Remove unused manifest scopes or add tool metadata showing why they are required.
    Related finding(s): fp_609d62f4dc434961
  • Manual review for SHIP-POLICY-APPROVAL-MISSING — Declare an approval policy for stripe.create_refund or remove this tool from the release.
    Related finding(s): fp_973ea0ef2110ca9a
  • Manual review for SHIP-POLICY-CONFIRMATION-MISSING — Declare a user confirmation policy for stripe.create_refund or remove this action from the release.
    Related finding(s): fp_c762eebfadaf39d9, fp_fae2921fd2d0cbd5
  • Manual review for SHIP-SCHEMA-BROAD-FREE-TEXT — Constrain zendesk.update_ticket.updates with an enum, structured schema, or narrower field-specific parameters.
    Related finding(s): fp_d58657d9b1c91a84, fp_ea4f18e4707f8e1d
  • Manual review for SHIP-SCHEMA-MISSING-BOUNDS — Add a maximum bound to stripe.create_refund.amount or document an equivalent limit in the tool policy.
    Related finding(s): fp_60fdf92126ba8844
  • Manual review for SHIP-SCOPE-PROHIBITED-TOOL-PRESENT — Remove stripe.create_refund, narrow its policy, or revise prohibited_actions so the manifest and tool surface do not contradict each other.
    Related finding(s): fp_5faaaee860bcec8b, fp_6f8e1ebab65f3607
  • Manual review for SHIP-SIDEFX-IDEMPOTENCY-MISSING — Add an idempotency key, idempotent annotation, or declared idempotency policy for stripe.create_refund.
    Related finding(s): fp_2cf0d6c77d9c3eee
  • Re-run scan after resolving source warnings — Source loaders emitted warnings; some tool surfaces may have been parsed with reduced confidence.

§10 What this packet did NOT prove

Agents Shipgate is an advisory tool: the deterministic merge gate for AI-generated agent capability changes, run as a local-first, static Tool-Use Readiness review. The packet below is derived from a scan; it does not, by itself, prove the following properties:

  • Prompt robustness. Whether the agent's prompt holds up under jailbreaks, persona drift, indirect prompt injection, or adversarial inputs.
  • Runtime behavior. Whether the agent actually invokes only the declared tools, respects approval gates at runtime, or follows policy under load. Static config is not runtime evidence.
  • Model correctness. Whether the underlying model produces correct outputs, calls the right tools, or stays within the declared scope. The packet does not benchmark the model.
  • Adversarial resistance. Whether the agent withstands red-team or penetration testing. The packet does not run scenarios; it organizes evidence.

Per-run residuals

  • Source warnings:
    • MCP source declares wildcard tool exposure
  • Low-confidence tool extractions: none
  • Suppressed findings in effect: none
  • Memory isolation is not modeled by the v0.1 manifest schema; no static evidence is available.
  • 5 active finding(s) came from heuristic provenance (keyword_heuristic=5, regex_heuristic=0); review the finding evidence before acting.
diff --git a/samples/support_refund_agent/expected/packet.json b/samples/support_refund_agent/expected/packet.json index d2194b4c..1b340f1a 100644 --- a/samples/support_refund_agent/expected/packet.json +++ b/samples/support_refund_agent/expected/packet.json @@ -69,8 +69,7 @@ "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", - "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", - "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac" + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" ], "tools": [ "shopify.cancel_order", @@ -79,8 +78,7 @@ "refund_status_lookup", "stripe.create_refund", "zendesk.update_ticket", - "gmail.send_customer_email", - "wildcard_mcp_tools.*" + "gmail.send_customer_email" ] }, "approval_coverage": { @@ -102,7 +100,7 @@ "pointer": "/policies/require_approval_for_tools", "ref": "shipgate.yaml#/policies/require_approval_for_tools", "start_column": null, - "start_line": 64, + "start_line": 82, "type": "manifest" }, "severity": "critical", @@ -218,8 +216,7 @@ "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", - "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", - "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac" + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" ], "observed_tools": [ "gmail.send_customer_email", @@ -228,7 +225,6 @@ "shopify.cancel_order", "stripe.create_refund", "support.search_kb", - "wildcard_mcp_tools.*", "zendesk.update_ticket" ], "prohibited_actions": [ @@ -255,7 +251,6 @@ "shopify.cancel_order", "stripe.create_refund", "support.search_kb", - "wildcard_mcp_tools.*", "zendesk.update_ticket" ] }, @@ -315,13 +310,6 @@ "scenario": "Manual review for SHIP-AUTH-SCOPE-COVERAGE-MISSING", "why": "Add the required scopes for shopify.cancel_order to permissions.scopes or narrow the tool's declared auth requirements." }, - { - "finding_ids": [ - "fp_a00e34cdb33a5596" - ], - "scenario": "Manual review for SHIP-INVENTORY-WILDCARD-TOOLS", - "why": "Replace wildcard tool exposure with an explicit tool allowlist before release review." - }, { "finding_ids": [ "fp_674fa79ae9993422" @@ -401,42 +389,16 @@ "rows": [ { "blocking_findings": [], - "confidence": "high", + "confidence": "medium", "domain": "Inventory", - "evidence_present": "partial", + "evidence_present": "covered", "evidence_source": [ "tool_inventory", "tool_surface", - "source_warnings", - "findings[]" - ], - "missing_controls": [ - "SHIP-INVENTORY-WILDCARD-TOOLS on wildcard_mcp_tools.*: Wildcard tool exposure declared" + "source_warnings" ], - "review_items": [ - { - "baseline_status": null, - "blocks_release": false, - "capability_refs": [], - "capability_trace_refs": [], - "check_id": "SHIP-INVENTORY-WILDCARD-TOOLS", - "fingerprint": "fp_a00e34cdb33a5596", - "id": "fp_a00e34cdb33a5596", - "policy_evidence_source": null, - "severity": "high", - "source": { - "end_line": null, - "location": null, - "path": ".agents-shipgate/wildcard-tools.json", - "pointer": "/wildcard", - "ref": ".agents-shipgate/wildcard-tools.json", - "start_column": null, - "start_line": null, - "type": "mcp" - }, - "title": "Wildcard tool exposure declared" - } - ] + "missing_controls": [], + "review_items": [] }, { "blocking_findings": [], @@ -557,7 +519,7 @@ "pointer": "/permissions/scopes", "ref": "shipgate.yaml", "start_column": null, - "start_line": 73, + "start_line": 91, "type": "manifest" }, "title": "Manifest declares broad permission scopes" @@ -693,7 +655,7 @@ "pointer": "/policies/require_approval_for_tools", "ref": "shipgate.yaml#/policies/require_approval_for_tools", "start_column": null, - "start_line": 64, + "start_line": 82, "type": "manifest" }, "severity": "critical", @@ -776,7 +738,7 @@ "pointer": "/policies/require_confirmation_for_tools", "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "start_column": null, - "start_line": 67, + "start_line": 85, "type": "manifest" }, "severity": "high", @@ -809,7 +771,7 @@ "pointer": "/policies/require_confirmation_for_tools", "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "start_column": null, - "start_line": 67, + "start_line": 85, "type": "manifest" }, "severity": "high", @@ -844,7 +806,7 @@ "pointer": "/policies/require_idempotency_for_tools", "ref": "shipgate.yaml#/policies/require_idempotency_for_tools", "start_column": null, - "start_line": 69, + "start_line": 87, "type": "manifest" }, "severity": "critical", @@ -916,7 +878,7 @@ "pointer": "/policies/require_approval_for_tools", "ref": "shipgate.yaml#/policies/require_approval_for_tools", "start_column": null, - "start_line": 64, + "start_line": 82, "type": "manifest" }, "severity": "critical", @@ -947,7 +909,7 @@ "pointer": "/policies/require_idempotency_for_tools", "ref": "shipgate.yaml#/policies/require_idempotency_for_tools", "start_column": null, - "start_line": 69, + "start_line": 87, "type": "manifest" }, "severity": "critical", @@ -1111,7 +1073,7 @@ "pointer": "/policies/require_confirmation_for_tools", "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "start_column": null, - "start_line": 67, + "start_line": 85, "type": "manifest" }, "severity": "high", @@ -1144,7 +1106,7 @@ "pointer": "/policies/require_confirmation_for_tools", "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "start_column": null, - "start_line": 67, + "start_line": 85, "type": "manifest" }, "severity": "high", @@ -1407,7 +1369,7 @@ "tool_id": "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8" } ], - "total_tools": 8 + "total_tools": 7 }, "human_in_the_loop": { "approval_required_tools": [ @@ -1452,7 +1414,7 @@ "pointer": "/policies/require_idempotency_for_tools", "ref": "shipgate.yaml#/policies/require_idempotency_for_tools", "start_column": null, - "start_line": 69, + "start_line": 87, "type": "manifest" }, "severity": "critical", @@ -1527,7 +1489,7 @@ } ] }, - "packet_schema_version": "0.9", + "packet_schema_version": "0.10", "project": { "name": "support-refund-agent", "owner": "support-platform", @@ -1559,7 +1521,7 @@ "pointer": "/policies/require_approval_for_tools", "ref": "shipgate.yaml#/policies/require_approval_for_tools", "start_column": null, - "start_line": 64, + "start_line": 82, "type": "manifest" }, "severity": "critical", @@ -1590,7 +1552,7 @@ "pointer": "/policies/require_idempotency_for_tools", "ref": "shipgate.yaml#/policies/require_idempotency_for_tools", "start_column": null, - "start_line": 69, + "start_line": 87, "type": "manifest" }, "severity": "critical", @@ -1675,90 +1637,209 @@ ], "decision": "blocked", "evidence_coverage": { + "binding_coverage": { + "gap_count": 1, + "pass_eligible": false, + "possible_tools": 0, + "reachable_tools": 7, + "reason_counts": { + "conflicting_binding_evidence": 1 + }, + "total_catalog_tools": 8, + "unbound_tools": 1 + }, "evidence_gaps": [ { - "kind": "incomplete_surface", + "kind": "conflicting_binding_evidence", + "next_action": { + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "auto_apply": false, + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "declaration_template": { + "agent_bindings": { + "declarations": [ + { + "agent": "root", + "complete": true, + "handoffs": [], + "reason": "", + "tools": [] + } + ], + "root": { + "object": "", + "source_id": "" + } + } + }, + "expects": "Reconcile the declaration with positive structural evidence and rerun verification.", + "kind": "resolve_binding_conflict", + "path": "shipgate.yaml#agent_bindings.declarations", + "requires_human_review": true, + "suggested_patch_kind": "manual", + "why": "A complete root-reachable static binding graph is required for passed." + }, + "source_ref": "/agent_bindings/declarations/0", + "source_type": null, + "subject": "agent_v1:7cb237a00d64b7400f4adc3b", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set." + }, + { + "kind": "conflicting_binding_evidence", "next_action": { "accepted_values": [ - "complete_mcp_export", - "openapi_spec", - "reviewed_explicit_inventory" + "match_structural_graph", + "correct_source_wiring" ], "auto_apply": false, "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", "declaration_template": null, - "expects": "Provide a complete MCP export, OpenAPI spec, or reviewed explicit tool inventory, then rerun verification.", - "kind": "provide_complete_inventory", - "path": "shipgate.yaml#tool_sources", + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "kind": "resolve_binding_conflict", + "path": "shipgate.yaml#agent_bindings", "requires_human_review": true, "suggested_patch_kind": "manual", - "why": "The complete statically-bound tool surface must be enumerable." + "why": "Conflicting binding evidence cannot be auto-resolved." }, - "source_ref": "/wildcard", + "source_ref": "/agent_bindings/declarations/0", "source_type": "mcp", - "subject": "wildcard_mcp_tools.* [wildcard_mcp_tools]", - "why": "tool surface enumeration is incomplete or not high confidence" + "subject": "gmail.send_customer_email [support_mcp_tools]", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set." }, { - "kind": "missing_authority_evidence", + "kind": "conflicting_binding_evidence", "next_action": { "accepted_values": [ - "none", - "scoped", - "unscoped", - "ambient" + "match_structural_graph", + "correct_source_wiring" ], "auto_apply": false, "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", - "declaration_template": { - "authority": { - "mode": "" - }, - "tool": "wildcard_mcp_tools.*" - }, - "expects": "Declare reviewed authority under action_surface.actions in shipgate.yaml, then rerun verification.", - "kind": "declare_action_authority", - "path": "shipgate.yaml#action_surface.actions[tool='wildcard_mcp_tools.*']", + "declaration_template": null, + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "kind": "resolve_binding_conflict", + "path": "shipgate.yaml#agent_bindings", "requires_human_review": true, "suggested_patch_kind": "manual", - "why": "A complete authority mode and grant are required." + "why": "Conflicting binding evidence cannot be auto-resolved." }, - "source_ref": "/wildcard", + "source_ref": "/agent_bindings/declarations/0", + "source_type": "openapi", + "subject": "refund_status_lookup [support_openapi]", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set." + }, + { + "kind": "conflicting_binding_evidence", + "next_action": { + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "auto_apply": false, + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "declaration_template": null, + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "kind": "resolve_binding_conflict", + "path": "shipgate.yaml#agent_bindings", + "requires_human_review": true, + "suggested_patch_kind": "manual", + "why": "Conflicting binding evidence cannot be auto-resolved." + }, + "source_ref": "/agent_bindings/declarations/0", "source_type": "mcp", - "subject": "wildcard_mcp_tools.* [wildcard_mcp_tools]", - "why": "tool has no explicit or structural authority evidence" + "subject": "send_email_preview [openai_sdk_static]", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set." }, { - "kind": "missing_effect_evidence", + "kind": "conflicting_binding_evidence", "next_action": { "accepted_values": [ - "read", - "write", - "destructive", - "external_communication", - "financial_write", - "production_operation", - "privileged_data_access", - "code_execution", - "identity_access" + "match_structural_graph", + "correct_source_wiring" ], "auto_apply": false, "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", - "declaration_template": { - "effect": "", - "tool": "wildcard_mcp_tools.*" - }, - "expects": "Declare the conservative effect under action_surface.actions in shipgate.yaml, then rerun verification.", - "kind": "declare_action_effect", - "path": "shipgate.yaml#action_surface.actions[tool='wildcard_mcp_tools.*']", + "declaration_template": null, + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "kind": "resolve_binding_conflict", + "path": "shipgate.yaml#agent_bindings", + "requires_human_review": true, + "suggested_patch_kind": "manual", + "why": "Conflicting binding evidence cannot be auto-resolved." + }, + "source_ref": "/agent_bindings/declarations/0", + "source_type": "openapi", + "subject": "shopify.cancel_order [support_openapi]", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set." + }, + { + "kind": "conflicting_binding_evidence", + "next_action": { + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "auto_apply": false, + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "declaration_template": null, + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "kind": "resolve_binding_conflict", + "path": "shipgate.yaml#agent_bindings", "requires_human_review": true, "suggested_patch_kind": "manual", - "why": "A reviewed or structural effect is required for an evidence-backed pass." + "why": "Conflicting binding evidence cannot be auto-resolved." }, - "source_ref": "/wildcard", + "source_ref": "/agent_bindings/declarations/0", + "source_type": "openapi", + "subject": "stripe.create_refund [support_openapi]", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set." + }, + { + "kind": "conflicting_binding_evidence", + "next_action": { + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "auto_apply": false, + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "declaration_template": null, + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "kind": "resolve_binding_conflict", + "path": "shipgate.yaml#agent_bindings", + "requires_human_review": true, + "suggested_patch_kind": "manual", + "why": "Conflicting binding evidence cannot be auto-resolved." + }, + "source_ref": "/agent_bindings/declarations/0", "source_type": "mcp", - "subject": "wildcard_mcp_tools.* [wildcard_mcp_tools]", - "why": "MCP tool has no explicit or structural effect evidence" + "subject": "support.search_kb [support_mcp_tools]", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set." + }, + { + "kind": "conflicting_binding_evidence", + "next_action": { + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "auto_apply": false, + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "declaration_template": null, + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "kind": "resolve_binding_conflict", + "path": "shipgate.yaml#agent_bindings", + "requires_human_review": true, + "suggested_patch_kind": "manual", + "why": "Conflicting binding evidence cannot be auto-resolved." + }, + "source_ref": "/agent_bindings/declarations/0", + "source_type": "openapi", + "subject": "zendesk.update_ticket [support_openapi]", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set." }, { "kind": "source_warning", @@ -1784,25 +1865,23 @@ "identity_coverage": { "ambiguous_name_count": 0, "bound_tools": 1, - "canonical_tools": 8, + "canonical_tools": 7, "gap_count": 0, - "pass_eligible_tools": 8, + "pass_eligible_tools": 7, "reason_counts": {}, - "total_observations": 9 + "total_observations": 8 }, "level": "static", "low_confidence_tool_count": 0, "semantic_coverage": { - "gap_count": 3, - "pass_eligible_actions": 6, + "gap_count": 7, + "pass_eligible_actions": 0, "reason_counts": { - "incomplete_surface": 1, - "missing_authority_evidence": 1, - "missing_effect_evidence": 1, + "conflicting_binding_evidence": 7, "unscoped_authority": 1 }, "review_concern_count": 1, - "total_actions": 8 + "total_actions": 7 }, "source_warning_count": 1 }, @@ -1815,28 +1894,6 @@ }, "reason": "5 active findings block release.", "review_items": [ - { - "baseline_status": null, - "blocks_release": false, - "capability_refs": [], - "capability_trace_refs": [], - "check_id": "SHIP-INVENTORY-WILDCARD-TOOLS", - "fingerprint": "fp_a00e34cdb33a5596", - "id": "fp_a00e34cdb33a5596", - "policy_evidence_source": null, - "severity": "high", - "source": { - "end_line": null, - "location": null, - "path": ".agents-shipgate/wildcard-tools.json", - "pointer": "/wildcard", - "ref": ".agents-shipgate/wildcard-tools.json", - "start_column": null, - "start_line": null, - "type": "mcp" - }, - "title": "Wildcard tool exposure declared" - }, { "baseline_status": null, "blocks_release": false, @@ -1920,7 +1977,7 @@ "pointer": "/permissions/scopes", "ref": "shipgate.yaml", "start_column": null, - "start_line": 73, + "start_line": 91, "type": "manifest" }, "title": "Manifest declares broad permission scopes" @@ -2092,7 +2149,7 @@ "pointer": "/policies/require_confirmation_for_tools", "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "start_column": null, - "start_line": 67, + "start_line": 85, "type": "manifest" }, "severity": "high", @@ -2125,7 +2182,7 @@ "pointer": "/policies/require_confirmation_for_tools", "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "start_column": null, - "start_line": 67, + "start_line": 85, "type": "manifest" }, "severity": "high", @@ -2191,7 +2248,7 @@ "static_verdict_disclaimer": "This verdict covers deterministic static evidence only. Agents Shipgate did not execute the agent or prove runtime behavior, tool routing, credential enforcement, or safety.", "verdict": "BLOCKED" }, - "run_id": "agents_shipgate_7429a94499efb630", + "run_id": "agents_shipgate_5ad1a92a984aacac", "scope_coverage": { "declared_scopes": [ "zendesk:tickets:read", diff --git a/samples/support_refund_agent/expected/packet.md b/samples/support_refund_agent/expected/packet.md index 16b9385b..d6ffdd19 100644 --- a/samples/support_refund_agent/expected/packet.md +++ b/samples/support_refund_agent/expected/packet.md @@ -3,9 +3,9 @@ - Project: support-refund-agent - Agent: refund-assistant - Environment: production\_like -- Run id: agents\_shipgate\_7429a94499efb630 +- Run id: agents\_shipgate\_5ad1a92a984aacac - Generated at: 2026-01-01T00:00:00\+00:00 -- Packet schema: 0\.9 +- Packet schema: 0\.10 This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See §10 for what the packet does *not* prove. @@ -14,7 +14,7 @@ This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See - Decision: `blocked` - Reason: 5 active findings block release. - Blockers: 5 -- Review items: 15 +- Review items: 14 ### CI gate behavior (informational) @@ -23,34 +23,33 @@ This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See ### Static semantic coverage -- Pass-eligible actions: 6/8 -- Evidence gaps: 3 +- Pass-eligible actions: 0/7 +- Evidence gaps: 7 - Known authority review concerns: 1 -- Reasons: incomplete\_surface=1, missing\_authority\_evidence=1, missing\_effect\_evidence=1, unscoped\_authority=1 +- Reasons: conflicting\_binding\_evidence=7, unscoped\_authority=1 ### Blockers -- `SHIP-POLICY-APPROVAL-MISSING` (critical): stripe.create\_refund lacks a declared approval policy — `specs/support-tools.openapi.yaml:97` — `shipgate.yaml:64` -- `SHIP-SIDEFX-IDEMPOTENCY-MISSING` (critical): stripe.create\_refund lacks idempotency evidence — `specs/support-tools.openapi.yaml:97` — `shipgate.yaml:69` +- `SHIP-POLICY-APPROVAL-MISSING` (critical): stripe.create\_refund lacks a declared approval policy — `specs/support-tools.openapi.yaml:97` — `shipgate.yaml:82` +- `SHIP-SIDEFX-IDEMPOTENCY-MISSING` (critical): stripe.create\_refund lacks idempotency evidence — `specs/support-tools.openapi.yaml:97` — `shipgate.yaml:87` - `SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING` (high): gmail.send\_customer\_email has external communication capability without required controls — `.agents-shipgate/mcp-tools.json\#/tools/1` - `SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING` (high): stripe.create\_refund has external communication capability without required controls — `specs/support-tools.openapi.yaml:97` - `SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING` (critical): stripe.create\_refund has financial write capability without required controls — `specs/support-tools.openapi.yaml:97` ### Review items -- `SHIP-INVENTORY-WILDCARD-TOOLS` (high): Wildcard tool exposure declared — `.agents-shipgate/wildcard-tools.json\#/wildcard` - `SHIP-SCHEMA-MISSING-BOUNDS` (high): stripe.create\_refund.amount has no maximum bound — `specs/support-tools.openapi.yaml:97` - `SHIP-SCHEMA-BROAD-FREE-TEXT` (high): zendesk.update\_ticket accepts broad free-form action input — `specs/support-tools.openapi.yaml:142` - `SHIP-SCHEMA-BROAD-FREE-TEXT` (high): gmail.send\_customer\_email accepts broad free-form action input — `.agents-shipgate/mcp-tools.json\#/tools/1` -- `SHIP-AUTH-MANIFEST-BROAD-SCOPE` (high): Manifest declares broad permission scopes — `shipgate.yaml:73` +- `SHIP-AUTH-MANIFEST-BROAD-SCOPE` (high): Manifest declares broad permission scopes — `shipgate.yaml:91` - `SHIP-AUTH-SCOPE-COVERAGE-MISSING` (high): shopify.cancel\_order requires scopes not declared in the manifest — `specs/support-tools.openapi.yaml:116` - `SHIP-AUTH-SCOPE-COVERAGE-MISSING` (high): support.search\_kb requires scopes not declared in the manifest — `.agents-shipgate/mcp-tools.json\#/tools/0` - `SHIP-AUTH-MISSING-SCOPE` (high): refund\_status\_lookup lacks declared auth scopes — `specs/support-tools.openapi.yaml:72` - `SHIP-AUTH-SCOPE-COVERAGE-MISSING` (high): gmail.send\_customer\_email requires scopes not declared in the manifest — `.agents-shipgate/mcp-tools.json\#/tools/1` - `SHIP-SCOPE-PROHIBITED-TOOL-PRESENT` (high): stripe.create\_refund appears to overlap with a prohibited action — `specs/support-tools.openapi.yaml:97` — `shipgate.yaml:22` - `SHIP-SCOPE-PROHIBITED-TOOL-PRESENT` (high): gmail.send\_customer\_email appears to overlap with a prohibited action — `.agents-shipgate/mcp-tools.json\#/tools/1` — `shipgate.yaml:24` -- `SHIP-POLICY-CONFIRMATION-MISSING` (high): stripe.create\_refund lacks a declared confirmation policy — `specs/support-tools.openapi.yaml:97` — `shipgate.yaml:67` -- `SHIP-POLICY-CONFIRMATION-MISSING` (high): gmail.send\_customer\_email lacks a declared confirmation policy — `.agents-shipgate/mcp-tools.json\#/tools/1` — `shipgate.yaml:67` +- `SHIP-POLICY-CONFIRMATION-MISSING` (high): stripe.create\_refund lacks a declared confirmation policy — `specs/support-tools.openapi.yaml:97` — `shipgate.yaml:85` +- `SHIP-POLICY-CONFIRMATION-MISSING` (high): gmail.send\_customer\_email lacks a declared confirmation policy — `.agents-shipgate/mcp-tools.json\#/tools/1` — `shipgate.yaml:85` - `SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING` (high): shopify.cancel\_order is high-risk but has no owner — `specs/support-tools.openapi.yaml:116` - `SHIP-MANIFEST-UNUSED-SCOPE` (medium): Manifest declares unused permission scope zendesk:tickets:read — `shipgate.yaml` @@ -60,7 +59,7 @@ This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See | Domain | Evidence present | Evidence source | Confidence | Missing controls | Blocking findings | Review items | |---|---|---|---|---|---|---| -| Inventory | partial | tool\_inventory; tool\_surface; \+2 more | high | SHIP-INVENTORY-WILDCARD-TOOLS on wildcard\_mcp\_tools.\*: Wildcard tool exposure declared | — | SHIP-INVENTORY-WILDCARD-TOOLS \(high\) | +| Inventory | covered | tool\_inventory; tool\_surface; \+1 more | medium | — | — | — | | Schema | partial | tool\_surface\_facts.tools\[\].hashes; findings\[\] | mixed | SHIP-SCHEMA-MISSING-BOUNDS on stripe.create\_refund: stripe.create\_refund.amount has no maximum bound; SHIP-SCHEMA-BROAD-FREE-TEXT on zendesk.update\_ticket: zendesk.update\_ticket accepts broad free-form action input; \+1 more | — | SHIP-SCHEMA-MISSING-BOUNDS \(high\); SHIP-SCHEMA-BROAD-FREE-TEXT \(high\); \+1 more | | Auth | partial | tool\_surface\_facts.scopes; tool\_inventory\[\].auth\_scopes; \+1 more | mixed | SHIP-AUTH-MANIFEST-BROAD-SCOPE: Manifest declares broad permission scopes; SHIP-AUTH-SCOPE-COVERAGE-MISSING on shopify.cancel\_order: shopify.cancel\_order requires scopes not declared in the manifest; \+4 more | — | SHIP-AUTH-MANIFEST-BROAD-SCOPE \(high\); SHIP-AUTH-SCOPE-COVERAGE-MISSING \(high\); \+4 more | | Approval | partial | tool\_surface\_facts.controls\[kind=approval\_policy\]; findings\[\] | high | SHIP-POLICY-APPROVAL-MISSING on stripe.create\_refund: stripe.create\_refund lacks a declared approval policy; SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING on stripe.create\_refund: stripe.create\_refund has financial write capability without required controls | SHIP-POLICY-APPROVAL-MISSING \(critical\); SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING \(critical\) | — | @@ -93,7 +92,6 @@ This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See - shopify.cancel\_order - stripe.create\_refund - support.search\_kb -- wildcard\_mcp\_tools.\* - zendesk.update\_ticket ### Divergences @@ -103,7 +101,7 @@ This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See ## §3 High-risk tool surface — partial -- Total tools: 8 · High-risk: 3 +- Total tools: 7 · High-risk: 3 | Tool | Source | Risk tags | Approval | Idempotency | |---|---|---|---|---| @@ -212,8 +210,6 @@ This packet is a reviewer-shaped synthesis of a static Agents Shipgate scan. See - Related finding(s): fp\_519cb82f038efd10 - **Manual review for SHIP-AUTH-SCOPE-COVERAGE-MISSING** — Add the required scopes for shopify.cancel\_order to permissions.scopes or narrow the tool's declared auth requirements. - Related finding(s): fp\_095f3a5337124f6e, fp\_1fd01b4ed2e41d51, fp\_24d610b0d4324190 -- **Manual review for SHIP-INVENTORY-WILDCARD-TOOLS** — Replace wildcard tool exposure with an explicit tool allowlist before release review. - - Related finding(s): fp\_a00e34cdb33a5596 - **Manual review for SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING** — Declare an owner for each high-risk production tool in risk\_overrides.tools. - Related finding(s): fp\_674fa79ae9993422 - **Manual review for SHIP-MANIFEST-UNUSED-SCOPE** — Remove unused manifest scopes or add tool metadata showing why they are required. diff --git a/samples/support_refund_agent/expected/report.json b/samples/support_refund_agent/expected/report.json index 5d8fb08d..f9e8fbd4 100644 --- a/samples/support_refund_agent/expected/report.json +++ b/samples/support_refund_agent/expected/report.json @@ -1,7 +1,7 @@ { "schema_version": "0.1", - "report_schema_version": "0.30", - "run_id": "agents_shipgate_7429a94499efb630", + "report_schema_version": "0.31", + "run_id": "agents_shipgate_5ad1a92a984aacac", "manifest_dir": "/samples/support_refund_agent", "project": { "name": "support-refund-agent", @@ -41,8 +41,7 @@ "refund_status_lookup", "stripe.create_refund", "zendesk.update_ticket", - "gmail.send_customer_email", - "wildcard_mcp_tools.*" + "gmail.send_customer_email" ], "tool_ids": [ "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", @@ -51,8 +50,7 @@ "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", - "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", - "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac" + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" ], "handoffs": [], "guardrails": { @@ -78,7 +76,7 @@ "summary": { "status": "release_blockers_detected", "critical_count": 3, - "high_count": 16, + "high_count": 15, "medium_count": 1, "low_count": 0, "info_count": 0, @@ -113,7 +111,7 @@ "ref": "shipgate.yaml#/policies/require_approval_for_tools", "location": null, "path": "shipgate.yaml", - "start_line": 64, + "start_line": 82, "end_line": null, "start_column": null, "pointer": "/policies/require_approval_for_tools" @@ -146,7 +144,7 @@ "ref": "shipgate.yaml#/policies/require_idempotency_for_tools", "location": null, "path": "shipgate.yaml", - "start_line": 69, + "start_line": 87, "end_line": null, "start_column": null, "pointer": "/policies/require_idempotency_for_tools" @@ -222,28 +220,6 @@ } ], "review_items": [ - { - "id": "fp_a00e34cdb33a5596", - "fingerprint": "fp_a00e34cdb33a5596", - "check_id": "SHIP-INVENTORY-WILDCARD-TOOLS", - "severity": "high", - "title": "Wildcard tool exposure declared", - "baseline_status": null, - "blocks_release": false, - "source": { - "type": "mcp", - "ref": ".agents-shipgate/wildcard-tools.json", - "location": null, - "path": ".agents-shipgate/wildcard-tools.json", - "start_line": null, - "end_line": null, - "start_column": null, - "pointer": "/wildcard" - }, - "policy_evidence_source": null, - "capability_refs": [], - "capability_trace_refs": [] - }, { "id": "fp_60fdf92126ba8844", "fingerprint": "fp_60fdf92126ba8844", @@ -323,7 +299,7 @@ "ref": "shipgate.yaml", "location": null, "path": "shipgate.yaml", - "start_line": 73, + "start_line": 91, "end_line": null, "start_column": null, "pointer": "/permissions/scopes" @@ -505,7 +481,7 @@ "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "location": null, "path": "shipgate.yaml", - "start_line": 67, + "start_line": 85, "end_line": null, "start_column": null, "pointer": "/policies/require_confirmation_for_tools" @@ -538,7 +514,7 @@ "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "location": null, "path": "shipgate.yaml", - "start_line": 67, + "start_line": 85, "end_line": null, "start_column": null, "pointer": "/policies/require_confirmation_for_tools" @@ -600,21 +576,80 @@ "low_confidence_tool_count": 0, "evidence_gaps": [ { - "kind": "incomplete_surface", - "subject": "wildcard_mcp_tools.* [wildcard_mcp_tools]", + "kind": "conflicting_binding_evidence", + "subject": "agent_v1:7cb237a00d64b7400f4adc3b", + "source_type": null, + "source_ref": "/agent_bindings/declarations/0", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "next_action": { + "kind": "resolve_binding_conflict", + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "path": "shipgate.yaml#agent_bindings.declarations", + "why": "A complete root-reachable static binding graph is required for passed.", + "expects": "Reconcile the declaration with positive structural evidence and rerun verification.", + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "suggested_patch_kind": "manual", + "declaration_template": { + "agent_bindings": { + "root": { + "source_id": "", + "object": "" + }, + "declarations": [ + { + "agent": "root", + "complete": true, + "tools": [], + "handoffs": [], + "reason": "" + } + ] + } + }, + "auto_apply": false, + "requires_human_review": true + } + }, + { + "kind": "conflicting_binding_evidence", + "subject": "gmail.send_customer_email [support_mcp_tools]", "source_type": "mcp", - "source_ref": "/wildcard", - "why": "tool surface enumeration is incomplete or not high confidence", + "source_ref": "/agent_bindings/declarations/0", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "next_action": { + "kind": "resolve_binding_conflict", + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "path": "shipgate.yaml#agent_bindings", + "why": "Conflicting binding evidence cannot be auto-resolved.", + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "suggested_patch_kind": "manual", + "declaration_template": null, + "auto_apply": false, + "requires_human_review": true + } + }, + { + "kind": "conflicting_binding_evidence", + "subject": "refund_status_lookup [support_openapi]", + "source_type": "openapi", + "source_ref": "/agent_bindings/declarations/0", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set.", "next_action": { - "kind": "provide_complete_inventory", + "kind": "resolve_binding_conflict", "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", - "path": "shipgate.yaml#tool_sources", - "why": "The complete statically-bound tool surface must be enumerable.", - "expects": "Provide a complete MCP export, OpenAPI spec, or reviewed explicit tool inventory, then rerun verification.", + "path": "shipgate.yaml#agent_bindings", + "why": "Conflicting binding evidence cannot be auto-resolved.", + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", "accepted_values": [ - "complete_mcp_export", - "openapi_spec", - "reviewed_explicit_inventory" + "match_structural_graph", + "correct_source_wiring" ], "suggested_patch_kind": "manual", "declaration_template": null, @@ -623,62 +658,111 @@ } }, { - "kind": "missing_authority_evidence", - "subject": "wildcard_mcp_tools.* [wildcard_mcp_tools]", + "kind": "conflicting_binding_evidence", + "subject": "send_email_preview [openai_sdk_static]", "source_type": "mcp", - "source_ref": "/wildcard", - "why": "tool has no explicit or structural authority evidence", + "source_ref": "/agent_bindings/declarations/0", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set.", "next_action": { - "kind": "declare_action_authority", + "kind": "resolve_binding_conflict", "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", - "path": "shipgate.yaml#action_surface.actions[tool='wildcard_mcp_tools.*']", - "why": "A complete authority mode and grant are required.", - "expects": "Declare reviewed authority under action_surface.actions in shipgate.yaml, then rerun verification.", + "path": "shipgate.yaml#agent_bindings", + "why": "Conflicting binding evidence cannot be auto-resolved.", + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", "accepted_values": [ - "none", - "scoped", - "unscoped", - "ambient" + "match_structural_graph", + "correct_source_wiring" ], "suggested_patch_kind": "manual", - "declaration_template": { - "tool": "wildcard_mcp_tools.*", - "authority": { - "mode": "" - } - }, + "declaration_template": null, + "auto_apply": false, + "requires_human_review": true + } + }, + { + "kind": "conflicting_binding_evidence", + "subject": "shopify.cancel_order [support_openapi]", + "source_type": "openapi", + "source_ref": "/agent_bindings/declarations/0", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "next_action": { + "kind": "resolve_binding_conflict", + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "path": "shipgate.yaml#agent_bindings", + "why": "Conflicting binding evidence cannot be auto-resolved.", + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "suggested_patch_kind": "manual", + "declaration_template": null, + "auto_apply": false, + "requires_human_review": true + } + }, + { + "kind": "conflicting_binding_evidence", + "subject": "stripe.create_refund [support_openapi]", + "source_type": "openapi", + "source_ref": "/agent_bindings/declarations/0", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "next_action": { + "kind": "resolve_binding_conflict", + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "path": "shipgate.yaml#agent_bindings", + "why": "Conflicting binding evidence cannot be auto-resolved.", + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "suggested_patch_kind": "manual", + "declaration_template": null, "auto_apply": false, "requires_human_review": true } }, { - "kind": "missing_effect_evidence", - "subject": "wildcard_mcp_tools.* [wildcard_mcp_tools]", + "kind": "conflicting_binding_evidence", + "subject": "support.search_kb [support_mcp_tools]", "source_type": "mcp", - "source_ref": "/wildcard", - "why": "MCP tool has no explicit or structural effect evidence", + "source_ref": "/agent_bindings/declarations/0", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set.", "next_action": { - "kind": "declare_action_effect", + "kind": "resolve_binding_conflict", "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", - "path": "shipgate.yaml#action_surface.actions[tool='wildcard_mcp_tools.*']", - "why": "A reviewed or structural effect is required for an evidence-backed pass.", - "expects": "Declare the conservative effect under action_surface.actions in shipgate.yaml, then rerun verification.", + "path": "shipgate.yaml#agent_bindings", + "why": "Conflicting binding evidence cannot be auto-resolved.", + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", "accepted_values": [ - "read", - "write", - "destructive", - "external_communication", - "financial_write", - "production_operation", - "privileged_data_access", - "code_execution", - "identity_access" + "match_structural_graph", + "correct_source_wiring" ], "suggested_patch_kind": "manual", - "declaration_template": { - "tool": "wildcard_mcp_tools.*", - "effect": "" - }, + "declaration_template": null, + "auto_apply": false, + "requires_human_review": true + } + }, + { + "kind": "conflicting_binding_evidence", + "subject": "zendesk.update_ticket [support_openapi]", + "source_type": "openapi", + "source_ref": "/agent_bindings/declarations/0", + "why": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "next_action": { + "kind": "resolve_binding_conflict", + "command": "agents-shipgate verify --workspace . --config shipgate.yaml --ci-mode advisory --format json", + "path": "shipgate.yaml#agent_bindings", + "why": "Conflicting binding evidence cannot be auto-resolved.", + "expects": "Reconcile positive structural evidence and reviewed declarations, then rerun verification.", + "accepted_values": [ + "match_structural_graph", + "correct_source_wiring" + ], + "suggested_patch_kind": "manual", + "declaration_template": null, "auto_apply": false, "requires_human_review": true } @@ -704,25 +788,34 @@ } ], "semantic_coverage": { - "total_actions": 8, - "pass_eligible_actions": 6, - "gap_count": 3, + "total_actions": 7, + "pass_eligible_actions": 0, + "gap_count": 7, "review_concern_count": 1, "reason_counts": { - "incomplete_surface": 1, - "missing_authority_evidence": 1, - "missing_effect_evidence": 1, + "conflicting_binding_evidence": 7, "unscoped_authority": 1 } }, "identity_coverage": { - "total_observations": 9, - "canonical_tools": 8, + "total_observations": 8, + "canonical_tools": 7, "bound_tools": 1, - "pass_eligible_tools": 8, + "pass_eligible_tools": 7, "ambiguous_name_count": 0, "gap_count": 0, "reason_counts": {} + }, + "binding_coverage": { + "total_catalog_tools": 8, + "reachable_tools": 7, + "possible_tools": 0, + "unbound_tools": 1, + "pass_eligible": false, + "gap_count": 1, + "reason_counts": { + "conflicting_binding_evidence": 1 + } } }, "baseline_delta": { @@ -743,14 +836,6 @@ "runtime_behavior_verified": false, "static_verdict_disclaimer": "This verdict covers deterministic static evidence only. Agents Shipgate did not execute the agent or prove runtime behavior, tool routing, credential enforcement, or safety.", "contribution_rules": [ - { - "finding_id": "fp_a00e34cdb33a5596", - "fingerprint": "fp_a00e34cdb33a5596", - "check_id": "SHIP-INVENTORY-WILDCARD-TOOLS", - "category": "review_item", - "rule": "review_required", - "rationale": "requires_human_review=true (severity=high); routed to review_items." - }, { "finding_id": "fp_60fdf92126ba8844", "fingerprint": "fp_60fdf92126ba8844", @@ -944,6 +1029,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -1076,7 +1194,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "customer_communication", @@ -1134,6 +1252,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/2", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -1291,6 +1442,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/4", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -1426,7 +1610,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "destructive", @@ -1481,6 +1665,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/5", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -1636,7 +1853,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "external_write", @@ -1697,6 +1914,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -1781,7 +2031,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "read_only" @@ -1796,105 +2046,6 @@ "fp_24d610b0d4324190" ] }, - { - "id": "cap_1ee04a89c7e78ddb", - "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", - "tool_name": "wildcard_mcp_tools.*", - "source_type": "mcp", - "source_id": "wildcard_mcp_tools", - "source_ref": ".agents-shipgate/wildcard-tools.json", - "capability": "wildcard_tool_surface", - "effect": "write", - "semantic_assessment": { - "conservative_effect": "write", - "identity": { - "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", - "status": "structural", - "provider": "wildcard_mcp_tools", - "binding_id": null, - "primary_observation_id": "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298", - "observation_ids": [ - "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298" - ], - "claims": [ - { - "dimension": "identity", - "value": "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298", - "confidence": "high", - "provenance_kind": "static_declaration", - "source": "source_observation", - "source_pointer": "/wildcard", - "evidence": { - "source_type": "mcp", - "source_id": "wildcard_mcp_tools", - "native_locator": "wildcard_mcp_tools.*" - } - } - ], - "issues": [], - "pass_eligible": true - }, - "effect": { - "status": "protocol_default", - "confidence": "low", - "claims": [ - { - "dimension": "effect", - "value": "write", - "confidence": "low", - "provenance_kind": "static_declaration", - "source": "mcp_protocol_default", - "source_pointer": "/wildcard", - "evidence": { - "reason": "missing effect annotations" - } - } - ], - "issues": [ - { - "kind": "incomplete_surface", - "dimension": "effect", - "message": "tool surface enumeration is incomplete or not high confidence", - "source": "tool_extraction", - "source_pointer": "/wildcard" - }, - { - "kind": "missing_effect_evidence", - "dimension": "effect", - "message": "MCP tool has no explicit or structural effect evidence", - "source": "mcp_protocol_default", - "source_pointer": "/wildcard" - } - ] - }, - "authority": { - "status": "unknown", - "mode": "unknown", - "auth_type": null, - "credential_mode": null, - "scopes": [], - "claims": [], - "issues": [ - { - "kind": "missing_authority_evidence", - "dimension": "authority", - "message": "tool has no explicit or structural authority evidence", - "source": "mcp", - "source_pointer": "/wildcard" - } - ] - }, - "pass_eligible": false - }, - "risk_tags": [], - "auth_scopes": [], - "owner": null, - "included_reason": "referenced_by_high_finding", - "control_status": "unknown", - "related_findings": [ - "fp_a00e34cdb33a5596" - ] - }, { "id": "cap_0da3f29d7a7c1ee8", "tool_id": "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", @@ -1933,23 +2084,56 @@ "issues": [], "pass_eligible": true }, - "effect": { + "binding": { "status": "declared", "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33" + ], "claims": [ { - "dimension": "effect", - "value": "write", + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", "confidence": "high", "provenance_kind": "static_declaration", - "source": "action_scope", - "source_pointer": "/paths/~1tickets~1{ticket_id}/patch", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/3", "evidence": { - "scope": "zendesk:tickets:write" + "edge_type": "direct_tool", + "complete": true } - }, - { - "dimension": "effect", + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_scope", + "source_pointer": "/paths/~1tickets~1{ticket_id}/patch", + "evidence": { + "scope": "zendesk:tickets:write" + } + }, + { + "dimension": "effect", "value": "write", "confidence": "high", "provenance_kind": "static_declaration", @@ -2044,7 +2228,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "write" @@ -2236,22 +2420,6 @@ "gap": "stripe.create_refund.amount has no maximum bound.", "release_implication": "Release reviewers cannot verify blast-radius limits." }, - { - "id": "mis_9ba5bb0073223f6e", - "kind": "control_missing", - "severity": "high", - "tool_name": "wildcard_mcp_tools.*", - "capability_refs": [ - "cap_1ee04a89c7e78ddb" - ], - "intention_refs": [], - "finding_refs": [ - "fp_a00e34cdb33a5596" - ], - "policy_requirement": "Tool exposure must use an explicit reviewed allowlist.", - "gap": "Wildcard tool exposure declared.", - "release_implication": "Release review cannot bound which tools may be available at runtime." - }, { "id": "mis_7cbbb0b3afb9bf61", "kind": "control_missing", @@ -2486,7 +2654,7 @@ "decision": "blocked", "summary": "5 release-relevant finding(s) map to active release blockers; resolve required controls or remove the capability.", "blocker_misalignment_count": 5, - "review_misalignment_count": 15, + "review_misalignment_count": 14, "fail_policy": { "ci_mode": "advisory", "fail_on": [], @@ -2552,19 +2720,6 @@ "fp_674fa79ae9993422" ] }, - { - "id": "scn_3604642326a85265", - "scenario_type": "wildcard_inventory", - "title": "Explicit tool inventory review", - "given": "Exercise the release path for wildcard_mcp_tools.*.", - "expected_control": "The release exposes a static allowlist instead of wildcard or unbounded tools.", - "source_misalignments": [ - "mis_9ba5bb0073223f6e" - ], - "source_findings": [ - "fp_a00e34cdb33a5596" - ] - }, { "id": "scn_aabad183796937bd", "scenario_type": "confirmation", @@ -2620,13 +2775,13 @@ } ], "tool_surface": { - "total_tools": 8, + "total_tools": 7, "high_risk_tools": 3, "sources": { - "mcp": 4, + "mcp": 3, "openapi": 4 }, - "wildcard_tools": 1, + "wildcard_tools": 0, "missing_descriptions": 0 }, "tool_surface_facts": { @@ -2679,7 +2834,7 @@ "input_schema": "f4cfd236c5b193ec", "output_schema": "44136fa355b3678a", "parameters": "4f1df264bfa081ee", - "annotations": "1f9a7cc7c90c44b6" + "annotations": "cbcbcf3476705e05" } }, { @@ -2808,27 +2963,6 @@ "parameters": "2cf78ed8690bc21f", "annotations": "5f182f4baa6c10e4" } - }, - { - "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", - "name": "wildcard_mcp_tools.*", - "provider": "wildcard_mcp_tools", - "source_type": "mcp", - "source_id": "wildcard_mcp_tools", - "source_ref": ".agents-shipgate/wildcard-tools.json", - "risk_tags": [], - "auth_scopes": [], - "owner": null, - "extraction_confidence": "high", - "has_description": true, - "hashes": { - "source_ref": "eb3c07f3ee88bf82", - "description": "e2599399bd9c0f3e", - "input_schema": "44136fa355b3678a", - "output_schema": "44136fa355b3678a", - "parameters": "4f53cda18c2baa0c", - "annotations": "691ae41e79efee45" - } } ], "scopes": [ @@ -3019,7 +3153,7 @@ ] }, "action_surface_facts": { - "snapshot_version": "0.2", + "snapshot_version": "0.3", "actions": [ { "action_id": "agent:support-refund-agent/refund-assistant:action_v2_053ad7751660e41c082a5e2abdf8c206eeb1336154bc764256a667121508143d", @@ -3067,6 +3201,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/3", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -3178,7 +3345,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "writes_data" @@ -3263,6 +3430,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/4", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -3398,7 +3598,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "destructive", @@ -3484,6 +3684,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -3616,7 +3849,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "external_communication", @@ -3704,6 +3937,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/2", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -3891,6 +4157,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -3975,7 +4274,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "financial_write", @@ -4059,6 +4358,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/5", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -4214,7 +4546,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "external_communication", @@ -4317,6 +4649,51 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/6", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + }, + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agents/refund_agent.py", + "source_pointer": "agents/refund_agent.py:10", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -4404,7 +4781,7 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false }, "risk_tags": [ "external_communication", @@ -4443,163 +4820,164 @@ "policy_hash": "99aa27a5e2ab49d6", "risk_hash": "a2ddef8c5875b92d" } + } + ] + }, + "action_surface_diff": { + "enabled": false, + "base": { + "kind": "none", + "path": null, + "report_schema_version": null, + "baseline_schema_version": null + }, + "summary": { + "actions_added": 0, + "actions_removed": 0, + "actions_modified": 0, + "scope_expansions": 0, + "effect_escalations": 0, + "risk_tags_added": 0, + "approvals_removed": 0, + "safeguards_removed": 0, + "input_schema_expansions": 0, + "blocking_findings": 3 + }, + "added": [], + "removed": [], + "modified": [], + "notes": [ + "No action-surface comparison source was provided." + ] + }, + "binding_surface_facts": { + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "status": "conflicting", + "agents": [ + { + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "name": "refund_agent", + "source_id": "openai_sdk_static", + "source_ref": "agents/refund_agent.py", + "source_pointer": "agents/refund_agent.py:10" + } + ], + "tool_edges": [ + { + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/4", + "complete": true }, { - "action_id": "agent:support-refund-agent/refund-assistant:action_v2_bc72ce8215630f6c73ee7bf24b15a8eebd00f12276838ae7f44832adf45cea8a", - "agent_id": "agent:support-refund-agent/refund-assistant", - "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", - "tool_name": "wildcard_mcp_tools.*", - "provider": "wildcard_mcp_tools", - "source_type": "mcp", - "source_id": "wildcard_mcp_tools", - "source_ref": ".agents-shipgate/wildcard-tools.json", - "source_location": null, - "source_path": ".agents-shipgate/wildcard-tools.json", - "source_start_line": null, - "source_end_line": null, - "source_start_column": null, - "source_pointer": "/wildcard", - "operation": "wildcard_mcp_tools.*", - "effect": "write", - "semantic_assessment": { - "conservative_effect": "write", - "identity": { - "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", - "status": "structural", - "provider": "wildcard_mcp_tools", - "binding_id": null, - "primary_observation_id": "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298", - "observation_ids": [ - "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298" - ], - "claims": [ - { - "dimension": "identity", - "value": "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298", - "confidence": "high", - "provenance_kind": "static_declaration", - "source": "source_observation", - "source_pointer": "/wildcard", - "evidence": { - "source_type": "mcp", - "source_id": "wildcard_mcp_tools", - "native_locator": "wildcard_mcp_tools.*" - } - } - ], - "issues": [], - "pass_eligible": true - }, - "effect": { - "status": "protocol_default", - "confidence": "low", - "claims": [ - { - "dimension": "effect", - "value": "write", - "confidence": "low", - "provenance_kind": "static_declaration", - "source": "mcp_protocol_default", - "source_pointer": "/wildcard", - "evidence": { - "reason": "missing effect annotations" - } - } - ], - "issues": [ - { - "kind": "incomplete_surface", - "dimension": "effect", - "message": "tool surface enumeration is incomplete or not high confidence", - "source": "tool_extraction", - "source_pointer": "/wildcard" - }, - { - "kind": "missing_effect_evidence", - "dimension": "effect", - "message": "MCP tool has no explicit or structural effect evidence", - "source": "mcp_protocol_default", - "source_pointer": "/wildcard" - } - ] - }, - "authority": { - "status": "unknown", - "mode": "unknown", - "auth_type": null, - "credential_mode": null, - "scopes": [], - "claims": [], - "issues": [ - { - "kind": "missing_authority_evidence", - "dimension": "authority", - "message": "tool has no explicit or structural authority evidence", - "source": "mcp", - "source_pointer": "/wildcard" - } - ] - }, - "pass_eligible": false - }, - "risk_tags": [ - "writes_data" - ], - "required_scopes": [], - "approval_policy": { - "required": false, - "threshold": null - }, - "safeguards": { - "idempotency": false, - "audit_log": null, - "rollback": null, - "dry_run": null - }, - "evidence": { - "owner": null, - "runbook": null, - "approval_ticket": null - }, - "input_fields": [], - "required_input_fields": [], - "input_schema_hash": "28cc233c2ea84993", - "hashes": { - "identity_hash": "788f43ca0f679311", - "schema_hash": "28cc233c2ea84993", - "policy_hash": "99aa27a5e2ab49d6", - "risk_hash": "d72dca7b64c1bf8a" - } + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agents/refund_agent.py", + "source_pointer": "agents/refund_agent.py:10", + "complete": true + }, + { + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/6", + "complete": true + }, + { + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "complete": true + }, + { + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/2", + "complete": true + }, + { + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/5", + "complete": true + }, + { + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/3", + "complete": true + }, + { + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "edge_type": "direct_tool", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "complete": true } - ] + ], + "handoff_edges": [], + "reachable_tool_ids": [ + "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" + ], + "possible_tool_ids": [], + "unbound_tool_ids": [ + "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac" + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_id": null, + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false }, - "action_surface_diff": { + "binding_surface_diff": { "enabled": false, - "base": { - "kind": "none", - "path": null, - "report_schema_version": null, - "baseline_schema_version": null - }, - "summary": { - "actions_added": 0, - "actions_removed": 0, - "actions_modified": 0, - "scope_expansions": 0, - "effect_escalations": 0, - "risk_tags_added": 0, - "approvals_removed": 0, - "safeguards_removed": 0, - "input_schema_expansions": 0, - "blocking_findings": 3 - }, - "added": [], - "removed": [], - "modified": [], - "notes": [ - "No action-surface comparison source was provided." - ], - "tool_id": null + "base_report_schema_version": null, + "added_reachable_tool_ids": [], + "removed_reachable_tool_ids": [], + "added_handoffs": [], + "removed_handoffs": [], + "notes": [] }, "capability_runtime_evidence": { "enabled": false, @@ -4624,45 +5002,6 @@ "codex_plugin_surface": null, "baseline": null, "findings": [ - { - "id": "fp_a00e34cdb33a5596", - "fingerprint": "fp_a00e34cdb33a5596", - "check_id": "SHIP-INVENTORY-WILDCARD-TOOLS", - "title": "Wildcard tool exposure declared", - "severity": "high", - "category": "inventory", - "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", - "tool_name": "wildcard_mcp_tools.*", - "agent_id": "agent:support-refund-agent/refund-assistant", - "evidence": { - "source_id": "wildcard_mcp_tools", - "source_ref": ".agents-shipgate/wildcard-tools.json" - }, - "confidence": "high", - "provenance_kind": "static_declaration", - "source": { - "type": "mcp", - "ref": ".agents-shipgate/wildcard-tools.json", - "location": null, - "path": ".agents-shipgate/wildcard-tools.json", - "pointer": "/wildcard" - }, - "policy_evidence_source": null, - "capability_refs": [], - "capability_policy_evidence": null, - "policy_routing": null, - "capability_trace_refs": [], - "recommendation": "Replace wildcard tool exposure with an explicit tool allowlist before release review.", - "blocks_release": false, - "suppressed": false, - "suppression_reason": null, - "baseline_status": null, - "autofix_safe": false, - "requires_human_review": true, - "suggested_patch_kind": "manual", - "docs_url": "https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/checks.md#ship-inventory-wildcard-tools", - "agent_action": "escalate_to_human" - }, { "id": "fp_60fdf92126ba8844", "fingerprint": "fp_60fdf92126ba8844", @@ -4806,7 +5145,7 @@ "ref": "shipgate.yaml", "location": null, "path": "shipgate.yaml", - "start_line": 73, + "start_line": 91, "pointer": "/permissions/scopes" }, "policy_evidence_source": null, @@ -5152,7 +5491,7 @@ "ref": "shipgate.yaml#/policies/require_approval_for_tools", "location": null, "path": "shipgate.yaml", - "start_line": 64, + "start_line": 82, "end_line": null, "start_column": null, "pointer": "/policies/require_approval_for_tools" @@ -5212,12 +5551,13 @@ }, "hashes": { "identity_hash": "578e492c26fd4656", + "binding_hash": "3797b951d702db48", "effect_hash": "3c5583f4a13cb1e3", "authority_hash": "fd53df901af7c2ec", "control_hash": "66ca58b2f8d2e5bb", "schema_hash": "81d9efbadca5ea8c", "risk_hash": "b9302728ac0d5ac2", - "evidence_hash": "d3c762447bbdedfb" + "evidence_hash": "a1876a84a6357d6c" }, "matched_predicates": { "missing_approval_policy": true, @@ -5283,7 +5623,7 @@ "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "location": null, "path": "shipgate.yaml", - "start_line": 67, + "start_line": 85, "end_line": null, "start_column": null, "pointer": "/policies/require_confirmation_for_tools" @@ -5343,12 +5683,13 @@ }, "hashes": { "identity_hash": "578e492c26fd4656", + "binding_hash": "3797b951d702db48", "effect_hash": "3c5583f4a13cb1e3", "authority_hash": "fd53df901af7c2ec", "control_hash": "66ca58b2f8d2e5bb", "schema_hash": "81d9efbadca5ea8c", "risk_hash": "b9302728ac0d5ac2", - "evidence_hash": "d3c762447bbdedfb" + "evidence_hash": "a1876a84a6357d6c" }, "matched_predicates": { "missing_confirmation_policy": true, @@ -5411,7 +5752,7 @@ "ref": "shipgate.yaml#/policies/require_confirmation_for_tools", "location": null, "path": "shipgate.yaml", - "start_line": 67, + "start_line": 85, "end_line": null, "start_column": null, "pointer": "/policies/require_confirmation_for_tools" @@ -5469,12 +5810,13 @@ }, "hashes": { "identity_hash": "e9991ea05213172f", + "binding_hash": "bc550fe7894d443e", "effect_hash": "0b01d252d3c04ac9", "authority_hash": "4fac84f970e42bce", "control_hash": "1230dbbe2d78c7d5", "schema_hash": "8c76077daf4af98a", "risk_hash": "89d9cf142700dd69", - "evidence_hash": "d630c61e18dfbea6" + "evidence_hash": "8cbbe01d45d3da05" }, "matched_predicates": { "missing_confirmation_policy": true, @@ -5540,7 +5882,7 @@ "ref": "shipgate.yaml#/policies/require_idempotency_for_tools", "location": null, "path": "shipgate.yaml", - "start_line": 69, + "start_line": 87, "end_line": null, "start_column": null, "pointer": "/policies/require_idempotency_for_tools" @@ -5790,8 +6132,8 @@ "Add the required scopes for support.search_kb to permissions.scopes or narrow the tool's declared auth requirements." ], "generated_reports": { - "markdown": "expected/report.md", - "json": "expected/report.json" + "markdown": "/private/tmp/shipgate-golden-support_refund_agent/report.md", + "json": "/private/tmp/shipgate-golden-support_refund_agent/report.json" }, "loaded_policy_packs": [], "loaded_plugins": [], @@ -5851,6 +6193,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/4", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -5986,7 +6361,40 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/4", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false } }, { @@ -6054,6 +6462,51 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/6", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + }, + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agents/refund_agent.py", + "source_pointer": "agents/refund_agent.py:10", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -6141,7 +6594,52 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/6", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + }, + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agents/refund_agent.py", + "source_pointer": "agents/refund_agent.py:10", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false } }, { @@ -6196,6 +6694,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -6280,7 +6811,40 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false } }, { @@ -6333,6 +6897,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/2", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -6440,6 +7037,39 @@ "issues": [] }, "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/2", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false } }, { @@ -6498,26 +7128,59 @@ "issues": [], "pass_eligible": true }, - "effect": { + "binding": { "status": "declared", "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8" + ], "claims": [ { - "dimension": "effect", - "value": "write", + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", "confidence": "high", "provenance_kind": "static_declaration", - "source": "action_scope", - "source_pointer": "/paths/~1refunds/post", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/5", "evidence": { - "scope": "stripe:refunds:write" + "edge_type": "direct_tool", + "complete": true } - }, + } + ], + "issues": [ { - "dimension": "effect", - "value": "financial_write", - "confidence": "high", - "provenance_kind": "static_declaration", + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_scope", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "scope": "stripe:refunds:write" + } + }, + { + "dimension": "effect", + "value": "financial_write", + "confidence": "high", + "provenance_kind": "static_declaration", "source": "action_surface_declaration", "source_pointer": "action_surface.actions[tool='stripe.create_refund'].effect", "evidence": {} @@ -6653,7 +7316,40 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/5", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false } }, { @@ -6708,6 +7404,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/3", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -6819,7 +7548,40 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/3", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false } }, { @@ -6876,6 +7638,39 @@ "issues": [], "pass_eligible": true }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, "effect": { "status": "declared", "confidence": "high", @@ -7008,105 +7803,1769 @@ ], "issues": [] }, - "pass_eligible": true + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false } - }, + } + ], + "tool_catalog": [ { - "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", - "name": "wildcard_mcp_tools.*", - "provider": "wildcard_mcp_tools", + "tool_id": "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "name": "shopify.cancel_order", + "provider": "support_openapi", "observation_ids": [ - "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298" + "obs_v1_fc50761fdbf05db319a85b8ac0b828b8522967a4273d1c0f70d4cfabf0304300" + ], + "source_type": "openapi", + "source_ref": "specs/support-tools.openapi.yaml#/paths/~1orders~1{order_id}~1cancel/post", + "source_path": "specs/support-tools.openapi.yaml", + "source_start_line": 116, + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post", + "risk_tags": [ + "destructive", + "write" + ], + "risk_tag_confidence": { + "destructive": "high", + "write": "high" + }, + "auth_scopes": [ + "shopify:orders:write" ], - "source_type": "mcp", - "source_ref": ".agents-shipgate/wildcard-tools.json", - "source_path": ".agents-shipgate/wildcard-tools.json", - "source_start_line": null, - "source_pointer": "/wildcard", - "risk_tags": [], - "risk_tag_confidence": {}, - "auth_scopes": [], "owner": null, "confidence": "high", "semantic_assessment": { - "conservative_effect": "write", + "conservative_effect": "destructive", "identity": { - "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", + "tool_id": "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", "status": "structural", - "provider": "wildcard_mcp_tools", + "provider": "support_openapi", "binding_id": null, - "primary_observation_id": "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298", + "primary_observation_id": "obs_v1_fc50761fdbf05db319a85b8ac0b828b8522967a4273d1c0f70d4cfabf0304300", "observation_ids": [ - "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298" + "obs_v1_fc50761fdbf05db319a85b8ac0b828b8522967a4273d1c0f70d4cfabf0304300" ], "claims": [ { "dimension": "identity", - "value": "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298", + "value": "obs_v1_fc50761fdbf05db319a85b8ac0b828b8522967a4273d1c0f70d4cfabf0304300", "confidence": "high", "provenance_kind": "static_declaration", "source": "source_observation", - "source_pointer": "/wildcard", + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post", "evidence": { - "source_type": "mcp", - "source_id": "wildcard_mcp_tools", - "native_locator": "wildcard_mcp_tools.*" + "source_type": "openapi", + "source_id": "support_openapi", + "native_locator": "POST /orders/{order_id}/cancel" } } ], "issues": [], "pass_eligible": true }, - "effect": { - "status": "protocol_default", - "confidence": "low", + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e" + ], "claims": [ { - "dimension": "effect", - "value": "write", - "confidence": "low", + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "confidence": "high", "provenance_kind": "static_declaration", - "source": "mcp_protocol_default", - "source_pointer": "/wildcard", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/4", "evidence": { - "reason": "missing effect annotations" + "edge_type": "direct_tool", + "complete": true } } ], "issues": [ { - "kind": "incomplete_surface", + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_scope", + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post", + "evidence": { + "scope": "shopify:orders:write" + } + }, + { + "dimension": "effect", + "value": "destructive", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='shopify.cancel_order'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "auth_scope", + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post", + "evidence": { + "scope": "shopify:orders:write" + } + }, + { + "dimension": "effect", + "value": "destructive", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post", + "evidence": { + "destructiveHint": true + } + }, + { "dimension": "effect", - "message": "tool surface enumeration is incomplete or not high confidence", - "source": "tool_extraction", - "source_pointer": "/wildcard" + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_method", + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post", + "evidence": { + "method": "POST" + } }, { - "kind": "missing_effect_evidence", "dimension": "effect", - "message": "MCP tool has no explicit or structural effect evidence", - "source": "mcp_protocol_default", - "source_pointer": "/wildcard" + "value": "destructive", + "confidence": "medium", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post", + "evidence": { + "tag": "destructive", + "hint_source": "keyword", + "method": "POST" + } } - ] + ], + "issues": [] }, "authority": { - "status": "unknown", - "mode": "unknown", - "auth_type": null, - "credential_mode": null, - "scopes": [], - "claims": [], - "issues": [ + "status": "declared", + "mode": "scoped", + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "shopify:orders:write" + ], + "claims": [ { - "kind": "missing_authority_evidence", "dimension": "authority", - "message": "tool has no explicit or structural authority evidence", - "source": "mcp", - "source_pointer": "/wildcard" - } - ] - }, + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='shopify.cancel_order'].authority", + "evidence": { + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "shopify:orders:write" + ] + } + }, + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_authority", + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post", + "evidence": { + "auth_type": "oauth2", + "scopes": [ + "shopify:orders:write" + ], + "alternative_count": 1 + } + }, + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_security_alternative", + "source_pointer": "/paths/~1orders~1{order_id}~1cancel/post/security/0", + "evidence": { + "anonymous": false, + "schemes": [ + { + "name": "shopifyOAuth", + "type": "oauth2", + "scopes": [ + "shopify:orders:write" + ] + } + ] + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_08f6c9642ff3da07bd4dd33f949b747ad5fe79c245d1f76d450a81ff86cb1f6e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/4", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + } + }, + { + "tool_id": "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "name": "send_email_preview", + "provider": "openai_sdk_static", + "observation_ids": [ + "obs_v1_68344b9fde158f18db1f3476cf06003414968e6d82488d8f376eebfecbc8bfb3", + "obs_v1_8835a4a63d13e804fae8ab9c175ece24e865d923e65457698593c0532254b0a6" + ], + "source_type": "mcp", + "source_ref": "inventories/sdk-tools.json", + "source_path": "inventories/sdk-tools.json", + "source_start_line": null, + "source_pointer": "/tools/0", + "risk_tags": [ + "read_only" + ], + "risk_tag_confidence": { + "read_only": "high" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "read", + "identity": { + "tool_id": "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "status": "declared", + "provider": "openai_sdk_static", + "binding_id": "sdk_send_email_preview", + "primary_observation_id": "obs_v1_8835a4a63d13e804fae8ab9c175ece24e865d923e65457698593c0532254b0a6", + "observation_ids": [ + "obs_v1_68344b9fde158f18db1f3476cf06003414968e6d82488d8f376eebfecbc8bfb3", + "obs_v1_8835a4a63d13e804fae8ab9c175ece24e865d923e65457698593c0532254b0a6" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_68344b9fde158f18db1f3476cf06003414968e6d82488d8f376eebfecbc8bfb3", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "agents/refund_agent.py", + "evidence": { + "source_type": "sdk_function", + "source_id": "openai_sdk_static", + "native_locator": "agents/refund_agent.py#send_email_preview" + } + }, + { + "dimension": "identity", + "value": "obs_v1_8835a4a63d13e804fae8ab9c175ece24e865d923e65457698593c0532254b0a6", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "tool_identity_binding", + "source_pointer": "/tools/0", + "evidence": { + "source_type": "mcp", + "source_id": "reviewed_sdk_inventory", + "native_locator": "send_email_preview" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/6", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + }, + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agents/refund_agent.py", + "source_pointer": "agents/refund_agent.py:10", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='send_email_preview'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/tools/0", + "evidence": { + "readOnlyHint": true + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "low", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/tools/0", + "evidence": { + "tag": "customer_communication", + "hint_source": "keyword" + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "low", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/tools/0", + "evidence": { + "tag": "external_write", + "hint_source": "keyword" + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "none", + "auth_type": null, + "credential_mode": null, + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='send_email_preview'].authority", + "evidence": { + "auth_type": null, + "credential_mode": null, + "scopes": [] + } + }, + { + "dimension": "authority", + "value": "none", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_authority", + "source_pointer": "/tools/0", + "evidence": { + "auth_type": null, + "scopes": [], + "alternative_count": 0 + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/6", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + }, + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_2c9ee6ae2b9f0259ba2dad40a30fb18a8bdc3a732ffc207936fdd49c80e1bc9e", + "confidence": "high", + "provenance_kind": "ast_extraction", + "source": "agents/refund_agent.py", + "source_pointer": "agents/refund_agent.py:10", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + } + }, + { + "tool_id": "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "name": "support.search_kb", + "provider": "support_mcp_tools", + "observation_ids": [ + "obs_v1_0df5fda3ee9f82ce968548fd66bcff348164c98aa8f25ca72cf8cea840ca4024" + ], + "source_type": "mcp", + "source_ref": ".agents-shipgate/mcp-tools.json", + "source_path": ".agents-shipgate/mcp-tools.json", + "source_start_line": null, + "source_pointer": "/tools/0", + "risk_tags": [ + "read_only" + ], + "risk_tag_confidence": { + "read_only": "high" + }, + "auth_scopes": [ + "support:kb:read" + ], + "owner": "support-platform", + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "read", + "identity": { + "tool_id": "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "status": "structural", + "provider": "support_mcp_tools", + "binding_id": null, + "primary_observation_id": "obs_v1_0df5fda3ee9f82ce968548fd66bcff348164c98aa8f25ca72cf8cea840ca4024", + "observation_ids": [ + "obs_v1_0df5fda3ee9f82ce968548fd66bcff348164c98aa8f25ca72cf8cea840ca4024" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_0df5fda3ee9f82ce968548fd66bcff348164c98aa8f25ca72cf8cea840ca4024", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "source_observation", + "source_pointer": "/tools/0", + "evidence": { + "source_type": "mcp", + "source_id": "support_mcp_tools", + "native_locator": "support.search_kb" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='support.search_kb'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/tools/0", + "evidence": { + "readOnlyHint": true + } + }, + { + "dimension": "effect", + "value": "financial_write", + "confidence": "low", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/tools/0", + "evidence": { + "tag": "financial_action", + "hint_source": "keyword", + "scopes": [ + "support:kb:read" + ] + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "scoped", + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "support:kb:read" + ], + "claims": [ + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='support.search_kb'].authority", + "evidence": { + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "support:kb:read" + ] + } + }, + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_authority", + "source_pointer": "/tools/0", + "evidence": { + "auth_type": "oauth2", + "scopes": [ + "support:kb:read" + ], + "alternative_count": 0 + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_445a251aa6adc1c69f7b318b68f7ad47f01b629d3c24d723331a661735cd353a", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/0", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + } + }, + { + "tool_id": "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "name": "refund_status_lookup", + "provider": "support_openapi", + "observation_ids": [ + "obs_v1_abe0748ca63b6b1f48d758ee5395281d80ad99b49e4766dda9f717c57aa6a3c0" + ], + "source_type": "openapi", + "source_ref": "specs/support-tools.openapi.yaml#/paths/~1refunds~1{payment_id}~1status/get", + "source_path": "specs/support-tools.openapi.yaml", + "source_start_line": 72, + "source_pointer": "/paths/~1refunds~1{payment_id}~1status/get", + "risk_tags": [ + "read_only" + ], + "risk_tag_confidence": { + "read_only": "high" + }, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "read", + "identity": { + "tool_id": "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "status": "structural", + "provider": "support_openapi", + "binding_id": null, + "primary_observation_id": "obs_v1_abe0748ca63b6b1f48d758ee5395281d80ad99b49e4766dda9f717c57aa6a3c0", + "observation_ids": [ + "obs_v1_abe0748ca63b6b1f48d758ee5395281d80ad99b49e4766dda9f717c57aa6a3c0" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_abe0748ca63b6b1f48d758ee5395281d80ad99b49e4766dda9f717c57aa6a3c0", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "source_observation", + "source_pointer": "/paths/~1refunds~1{payment_id}~1status/get", + "evidence": { + "source_type": "openapi", + "source_id": "support_openapi", + "native_locator": "GET /refunds/{payment_id}/status" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/2", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='refund_status_lookup'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_annotation", + "source_pointer": "/paths/~1refunds~1{payment_id}~1status/get", + "evidence": { + "readOnlyHint": true + } + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_method", + "source_pointer": "/paths/~1refunds~1{payment_id}~1status/get", + "evidence": { + "method": "GET" + } + }, + { + "dimension": "effect", + "value": "read", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "risk_hint:manual", + "source_pointer": "/paths/~1refunds~1{payment_id}~1status/get", + "evidence": { + "tag": "read_only", + "hint_source": "manual", + "reason": "Despite name containing refund, this is a read-only lookup.", + "confidence": "manual" + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "unscoped", + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [], + "claims": [ + { + "dimension": "authority", + "value": "unscoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='refund_status_lookup'].authority", + "evidence": { + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [] + } + }, + { + "dimension": "authority", + "value": "unscoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_authority", + "source_pointer": "/paths/~1refunds~1{payment_id}~1status/get", + "evidence": { + "auth_type": "oauth2", + "scopes": [], + "alternative_count": 1 + } + }, + { + "dimension": "authority", + "value": "unscoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_security_alternative", + "source_pointer": "/paths/~1refunds~1{payment_id}~1status/get/security/0", + "evidence": { + "anonymous": false, + "schemes": [ + { + "name": "stripeOAuth", + "type": "oauth2", + "scopes": [] + } + ] + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_61891eb768365da25408e00518646b94299be76989cba28407f47015112f86be", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/2", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + } + }, + { + "tool_id": "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "name": "stripe.create_refund", + "provider": "support_openapi", + "observation_ids": [ + "obs_v1_2dc88a16ab175fa6d937d4fd2047076fa2a6dd20131796d31e71b9998386eb57" + ], + "source_type": "openapi", + "source_ref": "specs/support-tools.openapi.yaml#/paths/~1refunds/post", + "source_path": "specs/support-tools.openapi.yaml", + "source_start_line": 97, + "source_pointer": "/paths/~1refunds/post", + "risk_tags": [ + "external_write", + "financial_action", + "write" + ], + "risk_tag_confidence": { + "external_write": "high", + "financial_action": "high", + "write": "high" + }, + "auth_scopes": [ + "stripe:refunds:write" + ], + "owner": "payments-platform", + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "financial_write", + "identity": { + "tool_id": "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "status": "structural", + "provider": "support_openapi", + "binding_id": null, + "primary_observation_id": "obs_v1_2dc88a16ab175fa6d937d4fd2047076fa2a6dd20131796d31e71b9998386eb57", + "observation_ids": [ + "obs_v1_2dc88a16ab175fa6d937d4fd2047076fa2a6dd20131796d31e71b9998386eb57" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_2dc88a16ab175fa6d937d4fd2047076fa2a6dd20131796d31e71b9998386eb57", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "source_observation", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "source_type": "openapi", + "source_id": "support_openapi", + "native_locator": "POST /refunds" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/5", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_scope", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "scope": "stripe:refunds:write" + } + }, + { + "dimension": "effect", + "value": "financial_write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='stripe.create_refund'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "auth_scope", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "scope": "stripe:refunds:write" + } + }, + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_method", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "method": "POST" + } + }, + { + "dimension": "effect", + "value": "financial_write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "risk_hint:auth_scope", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "tag": "financial_action", + "hint_source": "auth_scope", + "scopes": [ + "stripe:refunds:write" + ], + "method": "POST" + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "risk_hint:manual", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "tag": "external_write", + "hint_source": "manual", + "reason": "Creates real payment refunds.", + "confidence": "manual" + } + }, + { + "dimension": "effect", + "value": "financial_write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "risk_hint:manual", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "tag": "financial_action", + "hint_source": "manual", + "reason": "Creates real payment refunds.", + "confidence": "manual" + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "scoped", + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "stripe:refunds:write" + ], + "claims": [ + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='stripe.create_refund'].authority", + "evidence": { + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "stripe:refunds:write" + ] + } + }, + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_authority", + "source_pointer": "/paths/~1refunds/post", + "evidence": { + "auth_type": "oauth2", + "scopes": [ + "stripe:refunds:write" + ], + "alternative_count": 1 + } + }, + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_security_alternative", + "source_pointer": "/paths/~1refunds/post/security/0", + "evidence": { + "anonymous": false, + "schemes": [ + { + "name": "stripeOAuth", + "type": "oauth2", + "scopes": [ + "stripe:refunds:write" + ] + } + ] + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_8b559376c3236346b33a64fb9d460d342e6956579a68a0d5a331b2d2adf68ac8", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/5", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + } + }, + { + "tool_id": "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "name": "zendesk.update_ticket", + "provider": "support_openapi", + "observation_ids": [ + "obs_v1_0767af2d9bbe222adf666b94824bba1a94f3f2a146fa6d5198b2ef348da3fcc9" + ], + "source_type": "openapi", + "source_ref": "specs/support-tools.openapi.yaml#/paths/~1tickets~1{ticket_id}/patch", + "source_path": "specs/support-tools.openapi.yaml", + "source_start_line": 142, + "source_pointer": "/paths/~1tickets~1{ticket_id}/patch", + "risk_tags": [ + "write" + ], + "risk_tag_confidence": { + "write": "high" + }, + "auth_scopes": [ + "zendesk:tickets:write" + ], + "owner": null, + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "write", + "identity": { + "tool_id": "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "status": "structural", + "provider": "support_openapi", + "binding_id": null, + "primary_observation_id": "obs_v1_0767af2d9bbe222adf666b94824bba1a94f3f2a146fa6d5198b2ef348da3fcc9", + "observation_ids": [ + "obs_v1_0767af2d9bbe222adf666b94824bba1a94f3f2a146fa6d5198b2ef348da3fcc9" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_0767af2d9bbe222adf666b94824bba1a94f3f2a146fa6d5198b2ef348da3fcc9", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "source_observation", + "source_pointer": "/paths/~1tickets~1{ticket_id}/patch", + "evidence": { + "source_type": "openapi", + "source_id": "support_openapi", + "native_locator": "PATCH /tickets/{ticket_id}" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/3", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_scope", + "source_pointer": "/paths/~1tickets~1{ticket_id}/patch", + "evidence": { + "scope": "zendesk:tickets:write" + } + }, + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='zendesk.update_ticket'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "auth_scope", + "source_pointer": "/paths/~1tickets~1{ticket_id}/patch", + "evidence": { + "scope": "zendesk:tickets:write" + } + }, + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_method", + "source_pointer": "/paths/~1tickets~1{ticket_id}/patch", + "evidence": { + "method": "PATCH" + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "scoped", + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "zendesk:tickets:write" + ], + "claims": [ + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='zendesk.update_ticket'].authority", + "evidence": { + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "zendesk:tickets:write" + ] + } + }, + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_authority", + "source_pointer": "/paths/~1tickets~1{ticket_id}/patch", + "evidence": { + "auth_type": "oauth2", + "scopes": [ + "zendesk:tickets:write" + ], + "alternative_count": 1 + } + }, + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "openapi_security_alternative", + "source_pointer": "/paths/~1tickets~1{ticket_id}/patch/security/0", + "evidence": { + "anonymous": false, + "schemes": [ + { + "name": "zendeskOAuth", + "type": "oauth2", + "scopes": [ + "zendesk:tickets:write" + ] + } + ] + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_aa488250dc1213a6a8bb76a7e695da294f93052f3181d42817b6cd2cf83cfa33", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/3", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + } + }, + { + "tool_id": "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "name": "gmail.send_customer_email", + "provider": "support_mcp_tools", + "observation_ids": [ + "obs_v1_6a598e295d2d845f05a72faecd97fcf971b67c96a0aad092d9082c8e573df553" + ], + "source_type": "mcp", + "source_ref": ".agents-shipgate/mcp-tools.json", + "source_path": ".agents-shipgate/mcp-tools.json", + "source_start_line": null, + "source_pointer": "/tools/1", + "risk_tags": [ + "customer_communication", + "external_write" + ], + "risk_tag_confidence": { + "customer_communication": "high", + "external_write": "high" + }, + "auth_scopes": [ + "gmail:send" + ], + "owner": "support-platform", + "confidence": "high", + "semantic_assessment": { + "conservative_effect": "external_communication", + "identity": { + "tool_id": "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "status": "structural", + "provider": "support_mcp_tools", + "binding_id": null, + "primary_observation_id": "obs_v1_6a598e295d2d845f05a72faecd97fcf971b67c96a0aad092d9082c8e573df553", + "observation_ids": [ + "obs_v1_6a598e295d2d845f05a72faecd97fcf971b67c96a0aad092d9082c8e573df553" + ], + "claims": [ + { + "dimension": "identity", + "value": "obs_v1_6a598e295d2d845f05a72faecd97fcf971b67c96a0aad092d9082c8e573df553", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "source_observation", + "source_pointer": "/tools/1", + "evidence": { + "source_type": "mcp", + "source_id": "support_mcp_tools", + "native_locator": "gmail.send_customer_email" + } + } + ], + "issues": [], + "pass_eligible": true + }, + "binding": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + }, + "effect": { + "status": "declared", + "confidence": "high", + "claims": [ + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_scope", + "source_pointer": "/tools/1", + "evidence": { + "scope": "gmail:send" + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='gmail.send_customer_email'].effect", + "evidence": {} + }, + { + "dimension": "effect", + "value": "write", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "auth_scope", + "source_pointer": "/tools/1", + "evidence": { + "scope": "gmail:send" + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/tools/1", + "evidence": { + "tag": "customer_communication", + "hint_source": "keyword" + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "keyword_heuristic", + "source": "risk_hint:keyword", + "source_pointer": "/tools/1", + "evidence": { + "tag": "external_write", + "hint_source": "keyword" + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "risk_hint:manual", + "source_pointer": "/tools/1", + "evidence": { + "tag": "customer_communication", + "hint_source": "manual", + "reason": "Communicates refund status but cannot move funds.", + "confidence": "manual" + } + }, + { + "dimension": "effect", + "value": "external_communication", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "risk_hint:manual", + "source_pointer": "/tools/1", + "evidence": { + "tag": "external_write", + "hint_source": "manual", + "reason": "Communicates refund status but cannot move funds.", + "confidence": "manual" + } + } + ], + "issues": [] + }, + "authority": { + "status": "declared", + "mode": "scoped", + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "gmail:send" + ], + "claims": [ + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "action_surface_declaration", + "source_pointer": "action_surface.actions[tool='gmail.send_customer_email'].authority", + "evidence": { + "auth_type": "oauth2", + "credential_mode": "service_account", + "scopes": [ + "gmail:send" + ] + } + }, + { + "dimension": "authority", + "value": "scoped", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "mcp_authority", + "source_pointer": "/tools/1", + "evidence": { + "auth_type": "oauth2", + "scopes": [ + "gmail:send" + ], + "alternative_count": 0 + } + } + ], + "issues": [] + }, + "pass_eligible": false + }, + "binding_assessment": { + "status": "declared", + "confidence": "high", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [ + "agent_v1:7cb237a00d64b7400f4adc3b", + "tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9" + ], + "claims": [ + { + "dimension": "binding", + "value": "agent_v1:7cb237a00d64b7400f4adc3b->tool_v2_c69ebf76910efa80a2024a15d836f20366effdbd12f7a69b660ce4f4e5870ed9", + "confidence": "high", + "provenance_kind": "static_declaration", + "source": "agent_binding_declaration", + "source_pointer": "/agent_bindings/declarations/0/tools/1", + "evidence": { + "edge_type": "direct_tool", + "complete": true + } + } + ], + "issues": [ + { + "kind": "conflicting_binding_evidence", + "dimension": "binding", + "message": "Closed-world declaration for 'root' does not match the complete structural tool set.", + "source": "shipgate.yaml", + "source_pointer": "/agent_bindings/declarations/0" + } + ], + "pass_eligible": false + } + }, + { + "tool_id": "tool_v2_f21044a7610ceecc48cc135e8454cad369c1f60958e269277a05bc4812629fac", + "name": "wildcard_mcp_tools.*", + "provider": "wildcard_mcp_tools", + "observation_ids": [ + "obs_v1_7ab5a3762c9f308e499a7fde87c9569ded81c915ed7897560d87afa260a68298" + ], + "source_type": "mcp", + "source_ref": ".agents-shipgate/wildcard-tools.json", + "source_path": ".agents-shipgate/wildcard-tools.json", + "source_start_line": null, + "source_pointer": "/wildcard", + "risk_tags": [], + "risk_tag_confidence": {}, + "auth_scopes": [], + "owner": null, + "confidence": "high", + "semantic_assessment": null, + "binding_assessment": { + "status": "unknown", + "confidence": "low", + "root_agent_id": "agent_v1:7cb237a00d64b7400f4adc3b", + "reachable_path": [], + "claims": [], + "issues": [], "pass_eligible": false } } @@ -7116,11 +9575,11 @@ ], "agent_summary": { "verdict": "blocked", - "headline": "5 active finding(s) block release; 15 review item(s) accepted as debt.", + "headline": "5 active finding(s) block release; 14 review item(s) accepted as debt.", "blocker_count": 5, - "review_item_count": 15, + "review_item_count": 14, "auto_appliable_patches": 0, - "needs_human_review": 20, + "needs_human_review": 19, "first_recommended_action": { "kind": "info", "command": null, @@ -7156,9 +9615,9 @@ }, "reviewer_summary": { "verdict": "blocked", - "headline": "Release blocked: 20 lens changes. Start at release_decision.", + "headline": "Release blocked: 19 lens changes. Start at release_decision.", "tool_surface_changes": 0, - "capability_misalignments": 20, + "capability_misalignments": 19, "action_surface_changes": 0, "evidence_matrix_gaps": 0, "severity_overrides_applied": 0, @@ -7204,7 +9663,7 @@ "verdict": "blocked", "by_severity": { "critical": 3, - "high": 16, + "high": 15, "medium": 1 }, "by_reason_code": { @@ -7213,7 +9672,6 @@ "SHIP-AUTH-MANIFEST-BROAD-SCOPE": 1, "SHIP-AUTH-MISSING-SCOPE": 1, "SHIP-AUTH-SCOPE-COVERAGE-MISSING": 3, - "SHIP-INVENTORY-WILDCARD-TOOLS": 1, "SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING": 1, "SHIP-MANIFEST-UNUSED-SCOPE": 1, "SHIP-POLICY-APPROVAL-MISSING": 1, diff --git a/samples/support_refund_agent/expected/report.md b/samples/support_refund_agent/expected/report.md index 83b89d45..9999929f 100644 --- a/samples/support_refund_agent/expected/report.md +++ b/samples/support_refund_agent/expected/report.md @@ -16,8 +16,7 @@ Blockers (5): - HIGH SHIP-ACTION-EXTERNAL-COMMUNICATION-AUDIT-MISSING — stripe.create\_refund has external communication capability without required controls - CRITICAL SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING — stripe.create\_refund has financial write capability without required controls -Review items (15): -- HIGH SHIP-INVENTORY-WILDCARD-TOOLS — Wildcard tool exposure declared +Review items (14): - HIGH SHIP-SCHEMA-MISSING-BOUNDS — stripe.create\_refund.amount has no maximum bound - HIGH SHIP-SCHEMA-BROAD-FREE-TEXT — zendesk.update\_ticket accepts broad free-form action input - HIGH SHIP-SCHEMA-BROAD-FREE-TEXT — gmail.send\_customer\_email accepts broad free-form action input @@ -33,7 +32,7 @@ Review items (15): - HIGH SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING — shopify.cancel\_order is high-risk but has no owner - MEDIUM SHIP-MANIFEST-UNUSED-SCOPE — Manifest declares unused permission scope zendesk:tickets:read -Evidence coverage: static (1 source warning(s); 3 semantic evidence gap(s); 1 semantic review concern(s); 6/8 actions pass-eligible; human review recommended) +Evidence coverage: static (1 source warning(s); 1 binding evidence gap(s); 7/8 catalog tools reachable; 7 semantic evidence gap(s); 1 semantic review concern(s); 0/7 actions pass-eligible; human review recommended) Baseline delta: not enabled @@ -42,7 +41,7 @@ Fail policy: ci_mode=advisory, fail_on=[none], new_findings_only=false, would_fa ## Summary - Critical: 3 -- High: 16 +- High: 15 - Medium: 1 - Low: 0 - Suppressed: 0 @@ -76,7 +75,7 @@ Reviewer triage signal only. Provenance kind does not change severity, release d | Provenance kind | Active findings | | --- | ---: | -| `static_declaration` | 15 | +| `static_declaration` | 14 | | `ast_extraction` | 0 | | `keyword_heuristic` | 5 | | `regex_heuristic` | 0 | @@ -103,7 +102,7 @@ Actual capabilities: - shopify.cancel\_order: capability=destructive, risk=destructive, write, control=missing - stripe.create\_refund: capability=financial\_action, risk=external\_write, financial\_action, write, control=missing - support.search\_kb: capability=read\_only, risk=read\_only, control=missing -- 2 more in report.json +- 1 more in report.json Policy/control gaps: @@ -122,7 +121,7 @@ Policy/control gaps: - HIGH control\_missing \[shopify.cancel\_order\]: shopify.cancel\_order is high-risk but has no owner. (at specs/support-tools.openapi.yaml:116) Requires: Manifest metadata must match the active release surface. Release implication: Release review metadata is incomplete or stale. -- 15 more in report.json +- 14 more in report.json Release implication: @@ -135,8 +134,8 @@ Next validation: - Approval gate for high-risk action: The run records human approval before the tool call and denies calls without approval. - Tool schema boundary check: The tool accepts bounded structured inputs and returns structured outputs where needed. - High-risk tool validation case: A declared test or review scenario covers the high-risk tool path. -- Explicit tool inventory review: The release exposes a static allowlist instead of wildcard or unbounded tools. -- 3 more in report.json +- Confirmation gate for external or destructive action: The run records explicit confirmation before the side effect occurs. +- 2 more in report.json ## Recommended Next Actions @@ -155,11 +154,11 @@ Next validation: ## Tool Surface Summary -- Total tools: 8 +- Total tools: 7 - High-risk tools: 3 -- Wildcard tools: 1 +- Wildcard tools: 0 - Missing descriptions: 0 -- Sources: mcp=4, openapi=4 +- Sources: mcp=3, openapi=4 ## Action Surface Diff @@ -191,10 +190,6 @@ No local runtime trace artifacts were declared for capability evidence. - HIGH: SHIP-AUTH-SCOPE-COVERAGE-MISSING [shopify.cancel\_order] - shopify.cancel\_order requires scopes not declared in the manifest - HIGH: SHIP-AUTH-SCOPE-COVERAGE-MISSING [support.search\_kb] - support.search\_kb requires scopes not declared in the manifest -### Inventory - -- HIGH: SHIP-INVENTORY-WILDCARD-TOOLS [wildcard\_mcp\_tools.\*] - Wildcard tool exposure declared - ### Manifest - HIGH: SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING [shopify.cancel\_order] - shopify.cancel\_order is high-risk but has no owner @@ -221,7 +216,21 @@ No local runtime trace artifacts were declared for capability evidence. - CRITICAL: SHIP-SIDEFX-IDEMPOTENCY-MISSING [stripe.create\_refund] - stripe.create\_refund lacks idempotency evidence -## Appendix: Normalized Tool Inventory +## Agent Binding Surface + +Status: conflicting +Root agent: agent\_v1:7cb237a00d64b7400f4adc3b +Pass eligible: false +Catalog partition: 7 reachable, 0 possible, 1 unbound + +Binding gaps: +- `conflicting\_binding\_evidence` — Closed-world declaration for 'root' does not match the complete structural tool set. + +### Unbound Catalog Entries + +- `wildcard\_mcp\_tools.\*` (mcp) + +## Appendix: Root-Reachable Tool Inventory | Tool | Source | Risk Tags | Risk Confidence | Auth Scopes | Owner | | --- | --- | --- | --- | --- | --- | @@ -232,7 +241,6 @@ No local runtime trace artifacts were declared for capability evidence. | stripe.create\_refund | openapi | external\_write, financial\_action, write | external\_write=high, financial\_action=high, write=high | stripe:refunds:write | payments-platform | | zendesk.update\_ticket | openapi | write | write=high | zendesk:tickets:write | \- | | gmail.send\_customer\_email | mcp | customer\_communication, external\_write | customer\_communication=high, external\_write=high | gmail:send | support-platform | -| wildcard\_mcp\_tools.\* | mcp | \- | \- | \- | \- | ## Disclaimer diff --git a/samples/support_refund_agent/shipgate.yaml b/samples/support_refund_agent/shipgate.yaml index a2012fff..7e7cd3c5 100644 --- a/samples/support_refund_agent/shipgate.yaml +++ b/samples/support_refund_agent/shipgate.yaml @@ -60,6 +60,24 @@ tool_identity: - {source_id: openai_sdk_static, tool: send_email_preview} - {source_id: reviewed_sdk_inventory, tool: send_email_preview} +agent_bindings: + root: + source_id: openai_sdk_static + object: refund_agent + declarations: + - agent: root + complete: true + tools: + - {tool: support.search_kb, provider: support_mcp_tools} + - {tool: gmail.send_customer_email, provider: support_mcp_tools} + - {tool: refund_status_lookup, provider: support_openapi} + - {tool: zendesk.update_ticket, provider: support_openapi} + - {tool: shopify.cancel_order, provider: support_openapi} + - {tool: stripe.create_refund, provider: support_openapi} + - {tool: send_email_preview, provider: openai_sdk_static} + handoffs: [] + reason: reviewed multi-provider fixture binding + policies: require_approval_for_tools: - tool: shopify.cancel_order diff --git a/scripts/generate_schemas.py b/scripts/generate_schemas.py index 6f9101f9..2eec1bdb 100644 --- a/scripts/generate_schemas.py +++ b/scripts/generate_schemas.py @@ -43,10 +43,10 @@ - docs/host-grants-inventory-schema.v0.1.json (from agents_shipgate.schemas.host_grants. HostGrantsInventoryArtifactV1) -- docs/capability-lock-schema.v0.4.json +- docs/capability-lock-schema.v0.5.json (from agents_shipgate.schemas.capabilities. CapabilityLockFileArtifactV1) -- docs/capability-lock-diff-schema.v0.5.json +- docs/capability-lock-diff-schema.v0.6.json (from agents_shipgate.schemas.capabilities. CapabilityLockDiffArtifactV1) - docs/governance-benchmark-catalog-schema.v0.2.json diff --git a/scripts/run_governance_benchmark.py b/scripts/run_governance_benchmark.py index 4c9d8db9..445bc114 100644 --- a/scripts/run_governance_benchmark.py +++ b/scripts/run_governance_benchmark.py @@ -606,7 +606,28 @@ def _display_path(path: Path | None) -> str | None: return str(path) if not path.is_absolute() else path.name -def _manifest(*, environment: str = "production_like", ci: str = "advisory") -> str: +def _binding_declaration(tool_names: list[str]) -> str: + rendered_tools = "\n".join( + f" - tool: {tool_name}\n source_id: tools" + for tool_name in tool_names + ) + return f"""agent_bindings: + declarations: + - agent: root + complete: true + tools: +{rendered_tools if rendered_tools else " []"} + handoffs: [] + reason: independently reviewed governance benchmark binding +""" + + +def _manifest( + *, + environment: str = "production_like", + ci: str = "advisory", + tool_names: list[str] | None = None, +) -> str: return f"""version: "0.1" project: name: governance-benchmark @@ -623,7 +644,7 @@ def _manifest(*, environment: str = "production_like", ci: str = "advisory") -> trust: internal ci: mode: {ci} -""" +""" + _binding_declaration(tool_names or [_READ_TOOL["name"]]) def _manifest_with_scopes(scopes: list[str]) -> str: @@ -647,7 +668,7 @@ def _manifest_with_scopes(scopes: list[str]) -> str: {rendered_scopes} ci: mode: advisory -""" +""" + _binding_declaration([_BROAD_LIST_CASES_TOOL["name"]]) def _strict_manifest() -> str: @@ -670,7 +691,7 @@ def _strict_manifest() -> str: fail_on: - critical - high -""" +""" + _binding_declaration([_READ_TOOL["name"]]) def _tools(tools: list[dict[str, Any]]) -> str: @@ -788,7 +809,13 @@ def _added_tool_fixture(tool: dict[str, Any]) -> _FixtureTrees: base = _base_read_files() return _FixtureTrees( base_files=base, - head_files={"tools.json": _tools([_READ_TOOL, tool])}, + head_files={ + "shipgate.yaml": _manifest( + environment="local", + tool_names=[_READ_TOOL["name"], tool["name"]], + ), + "tools.json": _tools([_READ_TOOL, tool]), + }, ) @@ -811,10 +838,16 @@ def _mcp_safe_read_tool_added() -> _FixtureTrees: def _benign_remove_dangerous_tool() -> _FixtureTrees: return _FixtureTrees( base_files={ - "shipgate.yaml": _manifest(environment="local"), + "shipgate.yaml": _manifest( + environment="local", + tool_names=[_READ_TOOL["name"], _DELETE_TOOL["name"]], + ), "tools.json": _tools([_READ_TOOL, _DELETE_TOOL]), }, - head_files={"tools.json": _tools([_READ_TOOL])}, + head_files={ + "shipgate.yaml": _manifest(environment="local"), + "tools.json": _tools([_READ_TOOL]), + }, ) diff --git a/scripts/run_safety_qualification.py b/scripts/run_safety_qualification.py index f9343986..5fd5a7f7 100644 --- a/scripts/run_safety_qualification.py +++ b/scripts/run_safety_qualification.py @@ -331,6 +331,26 @@ def _evaluate_receipt( or semantic_coverage["pass_eligible_actions"] != semantic_coverage["total_actions"] ): raise ValueError("passed receipt violates evidence-backed pass invariants") + binding_coverage = evidence_coverage.get("binding_coverage") + if not isinstance(binding_coverage, dict): + raise ValueError("qualification report is missing binding_coverage") + required_binding_fields = { + "total_catalog_tools", + "reachable_tools", + "possible_tools", + "unbound_tools", + "pass_eligible", + "gap_count", + "reason_counts", + } + if not required_binding_fields.issubset(binding_coverage): + raise ValueError("qualification report binding_coverage is incomplete") + if receipt_decision == "passed" and ( + binding_coverage["gap_count"] != 0 + or binding_coverage["possible_tools"] != 0 + or binding_coverage["pass_eligible"] is not True + ): + raise ValueError("passed receipt violates binding-backed pass invariants") return receipt_decision, receipt_hash, [] except (OSError, UnicodeError, json.JSONDecodeError, ValidationError, ValueError) as exc: errors.append(str(exc)) diff --git a/scripts/verify_safety_qualification_release.py b/scripts/verify_safety_qualification_release.py index 9996fa4e..c7718dcf 100644 --- a/scripts/verify_safety_qualification_release.py +++ b/scripts/verify_safety_qualification_release.py @@ -337,9 +337,12 @@ def verify_release_qualification( expected_matrix_profiles = { "all", - "mcp", - "openapi", - "explicit_framework_inventory", + "mcp_openapi_declared_binding", + "openai_agents_sdk", + "langchain_crewai", + "google_adk", + "n8n", + "multi_agent_handoffs", "coding_agent_trust_roots", } matrices = {matrix.profile: matrix for matrix in result.confusion_matrices} diff --git a/skills/agents-shipgate/SKILL.md b/skills/agents-shipgate/SKILL.md index f072f512..00126def 100644 --- a/skills/agents-shipgate/SKILL.md +++ b/skills/agents-shipgate/SKILL.md @@ -76,9 +76,9 @@ For non-GitHub CI (GitLab, CircleCI, Jenkins, Azure Pipelines, Buildkite, Bitbuc - **Installed CLI contract**: when available, run `agents-shipgate contract --json` to verify local schema versions, capability/research surfaces, `release_decision.decision`, and manual-review signal fields. Older installs should use [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md) or upgrade before automating against the local contract command. - **Verifier JSON**: `verifier_schema_version: "0.2"`. Read `merge_verdict`, `can_merge_without_human`, `first_next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. - **Verify run JSON**: `verify-run.json` uses `schema_version: "shipgate.verify_run/v1"` and records stable run identity, subject refs, input hashes, outcome, and artifact hashes. It is the reproducibility artifact for `verify`; do not treat it as a second gate. -- **Report JSON**: `report_schema_version: "0.30"`. Read `release_decision.decision` first. A `passed` decision now requires complete, conflict-free static effect and authority evidence for every in-scope action, evaluation of all applicable controls, and no policy condition requiring review; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries; packet §1 mirrors them. Read `release_decision.evidence_coverage.semantic_coverage` and `identity_coverage`, then work every `evidence_gaps[].next_action` in order. Semantic gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; effect and authority declarations are human assertions and must never be auto-written. Use `agent_summary` / `findings[].agent_action` for routing and `reviewer_summary` for human review. To inspect heuristic findings without changing gate behavior, use `agents-shipgate findings --from agents-shipgate-reports/report.json --provenance-kind keyword_heuristic,regex_heuristic,runtime_trace --json`. Do not gate on legacy, baseline-blind `summary.status`. The current schema is [`docs/report-schema.v0.30.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.30.json); v0.29 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md#read-these-first-for-release-gating) and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). - These evidence-backed fields require runtime contract v12. A contract-v11 CLI emits the frozen v0.29 report and must not be described using the v0.30 identity claim. -- **Release Evidence Packet**: `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras) is a supporting/provisional reviewer artifact. Packet v0.9 projects semantic coverage and gap remediation under the release decision; it never creates a second gate. See [`docs/packet-schema.v0.9.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/packet-schema.v0.9.json) and [STABILITY.md §Release Evidence Packet](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md#release-evidence-packet-v09). +- **Report JSON**: `report_schema_version: "0.31"`. Read `release_decision.decision` first. A `passed` decision requires a complete root-reachable static binding graph plus complete, conflict-free identity, effect, and authority evidence for every reachable action; it does not prove runtime behavior. Preserve `release_decision.static_analysis_only=true`, `runtime_behavior_verified=false`, and `static_verdict_disclaimer` in summaries. Read `release_decision.evidence_coverage.binding_coverage`, `semantic_coverage`, and `identity_coverage`, then work every `evidence_gaps[].next_action` in order. Binding and semantic gaps are not Findings and cannot be suppressed, baselined, severity-overridden, cleared by `--no-heuristics`, or satisfied by `human_ack`; binding, effect, and authority declarations are human assertions and must never be auto-written. Use `tool_catalog[]` for diagnostics and `tool_inventory[]` for the proven reachable surface. The current schema is [`docs/report-schema.v0.31.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.31.json); v0.30 is a frozen compatibility reference. See the [current agent contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md#read-these-first-for-release-gating) and [evidence-backed passed contract](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/passed-verdict-contract.md). + These binding-backed fields require runtime contract v13. A contract-v12 CLI emits the frozen v0.30 report and must not be described using the v0.31 binding claim. +- **Release Evidence Packet**: `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras) is a supporting/provisional reviewer artifact. Packet v0.10 projects binding and semantic coverage under the release decision; it never creates a second gate. See [`docs/packet-schema.v0.10.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/packet-schema.v0.10.json) and [STABILITY.md §Release Evidence Packet](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md#release-evidence-packet-v010). - **Capability standard**: `agents-shipgate capability export` emits a stable static capability lock (`capability_lock_schema_version: "0.4"`) and `agents-shipgate capability diff` emits a stable semantic diff (`capability_lock_diff_schema_version: "0.5"`). These artifacts implement capability standard v0.3, are supporting/provisional and non-gating, exclude runtime trace evidence, and are documented in [`docs/capability-standard.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/capability-standard.md). - **Governance benchmark**: `benchmark/agent-pr-governance/cases.yaml` and `scripts/run_governance_benchmark.py` are the stable research benchmark substrate (`governance_benchmark_result_schema_version: "0.2"`), not a release gate. See [`docs/governance-benchmark.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/governance-benchmark.md). - **Single source of truth for the contract**: [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md). When the schema bumps, that file updates first. diff --git a/src/agents_shipgate/__init__.py b/src/agents_shipgate/__init__.py index 80d2ae6b..2561006d 100644 --- a/src/agents_shipgate/__init__.py +++ b/src/agents_shipgate/__init__.py @@ -1,3 +1,3 @@ """Agents Shipgate package.""" -__version__ = "0.16.0b1" +__version__ = "0.16.0b2" diff --git a/src/agents_shipgate/ci/release_decision.py b/src/agents_shipgate/ci/release_decision.py index a95d90aa..9c1d1759 100644 --- a/src/agents_shipgate/ci/release_decision.py +++ b/src/agents_shipgate/ci/release_decision.py @@ -10,6 +10,7 @@ from agents_shipgate.schemas.common import Severity from agents_shipgate.schemas.report import ( BaselineDelta, + BindingCoverageDecision, ContributionRule, ContributionRuleName, EvidenceCoverageDecision, @@ -56,6 +57,8 @@ def evidence_below_ie_threshold(evidence: EvidenceCoverageDecision, *, tool_coun non-mergeable verdicts they landed on. """ return ( + evidence.binding_coverage.gap_count > 0 + or evidence.semantic_coverage.gap_count > 0 or any( gap.kind == "source_warning" @@ -89,6 +92,7 @@ def build_release_decision( review_items: list[ReleaseDecisionItem] = [] contribution_rules: list[ContributionRule] = [] blocker_severities: set[Severity] = {"critical", *fail_on_resolved} + reachable_tool_ids = set(report.binding_surface_facts.reachable_tool_ids) # Phase 2c: an active (non-accepted) high/critical finding routed to # review is a *named* concern — the gate has decided "a human must look", # which is more specific than "insufficient_evidence". Tracked here so the @@ -175,7 +179,11 @@ def build_release_decision( or finding.requires_human_review is True ): review_items.append(_to_item(finding)) - if finding.severity in {"critical", "high"} and finding.baseline_status != "matched": + if ( + finding.severity in {"critical", "high"} + and finding.baseline_status != "matched" + and finding.tool_id in reachable_tool_ids + ): has_active_high_review = True contribution_rules.append( _rule( @@ -201,6 +209,7 @@ def build_release_decision( low_confidence_tool_count = sum(1 for tool in tools if tool.extraction_confidence != "high") semantic_coverage, semantic_gaps = _semantic_coverage(tools) identity_coverage = _identity_coverage(tools) + binding_coverage, binding_gaps = _binding_coverage(report) evidence = EvidenceCoverageDecision( level=report.summary.evidence_coverage, human_review_recommended=report.summary.human_review_recommended, @@ -208,9 +217,10 @@ def build_release_decision( low_confidence_tool_count=low_confidence_tool_count, # Semantic gaps lead because they are zero-tolerance gate inputs; # extraction/source gaps retain their existing deterministic order. - evidence_gaps=[*semantic_gaps, *_evidence_gaps(report, tools)], + evidence_gaps=[*binding_gaps, *semantic_gaps, *_evidence_gaps(report, tools)], semantic_coverage=semantic_coverage, identity_coverage=identity_coverage, + binding_coverage=binding_coverage, ) if report.baseline is None: @@ -226,14 +236,15 @@ def build_release_decision( evidence_is_degraded = evidence_below_ie_threshold(evidence, tool_count=len(tools)) has_semantic_gaps = semantic_coverage.gap_count > 0 + has_binding_gaps = binding_coverage.gap_count > 0 has_semantic_review_concerns = semantic_coverage.review_concern_count > 0 decision: ReleaseDecisionStatus if blockers: decision = "blocked" elif has_active_high_review: - # Phase 2c: a named, active high/critical concern (e.g. an unbounded - # toolkit mounted on the agent) is not "insufficient evidence" — the + # Phase 2c: a named, active high/critical concern on a proven-reachable + # capability is not "insufficient evidence" — the # gate HAS something concrete for a human to review. Prefer the # actionable review_required over the vaguer insufficient_evidence. # Both are equally non-auto-mergeable (can_merge_without_human=False), @@ -244,6 +255,8 @@ def build_release_decision( # a human via evidence_below_ie_threshold (the same predicate), so the # elevation never opens an auto-fix path on weak evidence. decision = "review_required" + elif has_binding_gaps: + decision = "insufficient_evidence" elif has_semantic_gaps: # v0.29: semantic gaps are not Findings. This makes them immune to # baselines, suppressions, severity overrides, human acknowledgement, @@ -279,7 +292,7 @@ def build_release_decision( fail_on=fail_on, new_findings_only=new_findings_only, release_decision=decision, - has_semantic_gaps=has_semantic_gaps, + has_semantic_gaps=has_semantic_gaps or has_binding_gaps, ) fail_policy = FailPolicy( ci_mode=ci_mode, @@ -350,6 +363,92 @@ def _is_mandatory_current_control(finding: Finding) -> bool: ) +def _binding_coverage( + report: ReadinessReport, +) -> tuple[BindingCoverageDecision, list[EvidenceGap]]: + graph = report.binding_surface_facts + reason_counts: dict[str, int] = {} + gaps: list[EvidenceGap] = [] + for issue in graph.issues: + _increment(reason_counts, issue.kind) + if issue.kind == "ambiguous_root_agent": + action_kind = "declare_agent_root" + path = "shipgate.yaml#agent_bindings.root" + accepted_values = ["source_id", "object"] + expects = "Declare the exact root agent object and rerun verification." + elif issue.kind in {"missing_binding_evidence", "unresolved_bound_tool"}: + action_kind = "declare_agent_bindings" + path = "shipgate.yaml#agent_bindings.declarations" + accepted_values = ["agent", "complete:true", "tools", "handoffs", "reason"] + expects = "Add a reviewed closed-world binding declaration and rerun verification." + elif issue.kind in {"partial_binding_evidence", "incomplete_handoff_graph"}: + action_kind = "provide_complete_binding_graph" + path = "shipgate.yaml#agent_bindings" + accepted_values = ["literal_tools", "literal_handoffs", "complete:true"] + expects = "Provide static wiring or an exact reviewed declaration for every reachable edge." + elif issue.kind == "conflicting_binding_evidence": + action_kind = "resolve_binding_conflict" + path = "shipgate.yaml#agent_bindings.declarations" + accepted_values = ["match_structural_graph", "correct_source_wiring"] + expects = "Reconcile the declaration with positive structural evidence and rerun verification." + elif issue.kind == "unresolved_agent_binding": + action_kind = "declare_agent_root" + path = "shipgate.yaml#agent_bindings" + accepted_values = ["exact_agent_object", "exact_handoff_target"] + expects = "Correct the agent selector or handoff target and rerun verification." + else: + action_kind = "provide_static_binding_source" + path = "shipgate.yaml#agent_bindings" + accepted_values = ["literal_binding", "reviewed_declaration"] + expects = "Correct the binding annotation or provide an exact reviewed declaration." + gaps.append( + EvidenceGap( + kind=issue.kind, + subject=issue.tool_id or issue.agent_id or graph.root_agent_id or "agent_binding_graph", + source_ref=issue.source_pointer or issue.source, + why=issue.message, + next_action=EvidenceGapAction( + kind=action_kind, # type: ignore[arg-type] + command=_SEMANTIC_RERUN_COMMAND, + path=path, + why="A complete root-reachable static binding graph is required for passed.", + expects=expects, + accepted_values=accepted_values, + declaration_template={ + "agent_bindings": { + "root": {"source_id": "", "object": ""}, + "declarations": [ + { + "agent": "root", + "complete": True, + "tools": [], + "handoffs": [], + "reason": "", + } + ], + } + }, + ), + ) + ) + return ( + BindingCoverageDecision( + total_catalog_tools=( + len(graph.reachable_tool_ids) + + len(graph.possible_tool_ids) + + len(graph.unbound_tool_ids) + ), + reachable_tools=len(graph.reachable_tool_ids), + possible_tools=len(graph.possible_tool_ids), + unbound_tools=len(graph.unbound_tool_ids), + pass_eligible=graph.pass_eligible, + gap_count=len(gaps), + reason_counts=dict(sorted(reason_counts.items())), + ), + gaps, + ) + + def _semantic_coverage( tools: list[Tool], ) -> tuple[SemanticCoverageDecision, list[EvidenceGap]]: @@ -378,6 +477,7 @@ def _semantic_coverage( issues = sorted( [ *assessment.identity.issues, + *assessment.binding.issues, *assessment.effect.issues, *assessment.authority.issues, ], @@ -522,6 +622,27 @@ def _semantic_gap( accepted_values = ["split_binding", "align_schema", "align_authority", "align_annotations"] action_why = "Conflicting bound observations cannot share one canonical capability." expects = "Split the binding or reconcile its structural evidence, then rerun verification." + elif kind in { + "missing_binding_evidence", + "partial_binding_evidence", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "ambiguous_root_agent", + }: + action_kind = ( + "declare_agent_root" + if kind in {"ambiguous_root_agent", "unresolved_agent_binding"} + else "declare_agent_bindings" + ) + accepted_values = ["root", "complete:true", "tools", "handoffs", "reason"] + action_why = "An exact root-reachable binding graph is required." + expects = "Provide static wiring or a reviewed closed-world declaration, then rerun verification." + elif kind in {"conflicting_binding_evidence", "invalid_binding_annotation"}: + action_kind = "resolve_binding_conflict" + accepted_values = ["match_structural_graph", "correct_source_wiring"] + action_why = "Conflicting binding evidence cannot be auto-resolved." + expects = "Reconcile positive structural evidence and reviewed declarations, then rerun verification." elif kind == "incomplete_surface": action_kind = "provide_complete_inventory" accepted_values = [ @@ -611,7 +732,20 @@ def _semantic_gap( "ambiguous_legacy_tool_identity", "invalid_tool_binding", } - else f"shipgate.yaml#action_surface.actions[tool={tool.name!r}]" + else ( + "shipgate.yaml#agent_bindings" + if kind in { + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation", + } + else f"shipgate.yaml#action_surface.actions[tool={tool.name!r}]" + ) ) ), why=action_why, diff --git a/src/agents_shipgate/cli/scan/decision.py b/src/agents_shipgate/cli/scan/decision.py index 11531be1..de8421a4 100644 --- a/src/agents_shipgate/cli/scan/decision.py +++ b/src/agents_shipgate/cli/scan/decision.py @@ -96,6 +96,8 @@ def _run_checks_and_decide( agent=tools_and_agent.agent, tools=tools_and_agent.tools, config_path=config_path.resolve(), + tool_catalog=tools_and_agent.tool_catalog, + binding_graph=tools_and_agent.binding_graph, framework_artifacts=inputs.artifact_bag, action_surface_facts=action_surface_facts, capability_facts=capability_facts, diff --git a/src/agents_shipgate/cli/scan/final_report.py b/src/agents_shipgate/cli/scan/final_report.py index 4c92846a..b8704e1d 100644 --- a/src/agents_shipgate/cli/scan/final_report.py +++ b/src/agents_shipgate/cli/scan/final_report.py @@ -51,6 +51,8 @@ def _build_final_report( frameworks=sanitized.frameworks_surface, codex_plugin_surface=sanitized.codex_plugin_surface, action_surface_facts=sanitized.action_surface_facts, + tool_catalog=sanitized.tool_catalog, + binding_graph=sanitized.binding_graph, ), manifest=sanitized.manifest, project=sanitized.project, @@ -58,6 +60,9 @@ def _build_final_report( agent=sanitized.agent.model_dump(exclude_none=True), environment=sanitized.environment, tools=sanitized.tools, + tool_catalog=sanitized.tool_catalog, + binding_surface_facts=sanitized.binding_graph, + binding_surface_diff=sanitized.binding_surface_diff, findings=sanitized.findings, generated_reports=plan.generated_report_refs, ci_mode=sanitized.manifest.ci.mode, diff --git a/src/agents_shipgate/cli/scan/models.py b/src/agents_shipgate/cli/scan/models.py index ef803baf..2c7baa8c 100644 --- a/src/agents_shipgate/cli/scan/models.py +++ b/src/agents_shipgate/cli/scan/models.py @@ -19,6 +19,7 @@ from agents_shipgate.core.domain import Agent, LoadedToolSource, Tool, ToolkitScopeBound from agents_shipgate.core.lenses.tool_surface import ToolSurfaceDiffReference from agents_shipgate.core.privacy import RedactionStats +from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment, BindingSurfaceDiff from agents_shipgate.schemas.codex_plugin import CodexPluginSurface from agents_shipgate.schemas.manifest import AgentsShipgateManifest from agents_shipgate.schemas.report import PolicyAudit @@ -78,7 +79,9 @@ class _ToolsAndAgent: """Phase 3 output: flattened/deduped/enriched tools + Agent + final warnings.""" tools: list[Tool] + tool_catalog: list[Tool] agent: Agent + binding_graph: AgentBindingGraphAssessment warnings: list[str] # deduplicated source warnings # Statically-parsed least-privilege bounds on dynamically-loaded # toolkits, aggregated across all loaded sources. Empty for the common @@ -151,6 +154,9 @@ class _SanitizedSurfaces: environment: Any agent: Agent tools: list[Tool] + tool_catalog: list[Tool] + binding_graph: AgentBindingGraphAssessment + binding_surface_diff: BindingSurfaceDiff findings: list[Any] source_warnings: list[str] api_artifacts: OpenAIApiArtifacts | None diff --git a/src/agents_shipgate/cli/scan/run_identity.py b/src/agents_shipgate/cli/scan/run_identity.py index e93324ee..5bdae455 100644 --- a/src/agents_shipgate/cli/scan/run_identity.py +++ b/src/agents_shipgate/cli/scan/run_identity.py @@ -5,6 +5,7 @@ from agents_shipgate.core.domain import Tool from agents_shipgate.core.findings.summaries import tool_inventory +from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment from agents_shipgate.schemas.codex_plugin import CodexPluginSurface from agents_shipgate.schemas.surfaces import ActionSurfaceFacts @@ -31,6 +32,8 @@ def _run_id( frameworks: dict[str, object] | None = None, codex_plugin_surface: CodexPluginSurface | None = None, action_surface_facts: ActionSurfaceFacts | None = None, + tool_catalog: list[Tool] | None = None, + binding_graph: AgentBindingGraphAssessment | None = None, ) -> str: payload = { "project": project @@ -41,6 +44,10 @@ def _run_id( if environment is not None else manifest.environment.model_dump(mode="json", exclude_none=False), "tool_inventory": tool_inventory(tools), + "tool_catalog": tool_inventory(tool_catalog if tool_catalog is not None else tools), + "binding_graph": ( + binding_graph.model_dump(mode="json") if binding_graph is not None else None + ), "findings": [ finding.model_dump( mode="json", diff --git a/src/agents_shipgate/cli/scan/sanitization.py b/src/agents_shipgate/cli/scan/sanitization.py index ffb37a07..3ab0da60 100644 --- a/src/agents_shipgate/cli/scan/sanitization.py +++ b/src/agents_shipgate/cli/scan/sanitization.py @@ -39,6 +39,7 @@ sanitize_model, sanitize_tools, ) +from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment, BindingSurfaceDiff from agents_shipgate.schemas.manifest import AgentsShipgateManifest from agents_shipgate.schemas.report import ( BaselineSummary, @@ -135,6 +136,47 @@ def _sanitize_for_output( else None ) public_tools = sanitize_tools(tools_and_agent.tools, stats=privacy_stats) + public_tool_catalog = sanitize_tools(tools_and_agent.tool_catalog, stats=privacy_stats) + public_binding_graph = sanitize_model( + tools_and_agent.binding_graph, + AgentBindingGraphAssessment, + stats=privacy_stats, + path="binding_surface_facts", + ) + base_binding = diffs.diff_reference.binding_facts if diffs.diff_reference else None + if base_binding is None: + public_binding_diff = BindingSurfaceDiff( + enabled=False, + base_report_schema_version=( + diffs.diff_reference.report_schema_version if diffs.diff_reference else None + ), + notes=[ + "Binding diff requires a report_schema_version 0.31 base report." + ] if diffs.diff_reference is not None else [], + ) + else: + current_handoffs = { + f"{edge.source_agent_id}->{edge.target_agent_id}:{edge.edge_type}" + for edge in public_binding_graph.handoff_edges + } + base_handoffs = { + f"{edge.source_agent_id}->{edge.target_agent_id}:{edge.edge_type}" + for edge in base_binding.handoff_edges + } + public_binding_diff = BindingSurfaceDiff( + enabled=True, + base_report_schema_version=diffs.diff_reference.report_schema_version, + added_reachable_tool_ids=sorted( + set(public_binding_graph.reachable_tool_ids) + - set(base_binding.reachable_tool_ids) + ), + removed_reachable_tool_ids=sorted( + set(base_binding.reachable_tool_ids) + - set(public_binding_graph.reachable_tool_ids) + ), + added_handoffs=sorted(current_handoffs - base_handoffs), + removed_handoffs=sorted(base_handoffs - current_handoffs), + ) public_findings = sanitize_findings(decision.findings, stats=privacy_stats) assign_finding_ids(public_findings) public_capability_runtime_evidence = sanitize_model( @@ -310,6 +352,9 @@ def _sanitize_for_output( environment=public_environment, agent=public_agent, tools=public_tools, + tool_catalog=public_tool_catalog, + binding_graph=public_binding_graph, + binding_surface_diff=public_binding_diff, findings=public_findings, source_warnings=public_source_warnings, api_artifacts=public_api_artifacts, diff --git a/src/agents_shipgate/cli/scan/tools_agent.py b/src/agents_shipgate/cli/scan/tools_agent.py index 9d7314ac..056bb507 100644 --- a/src/agents_shipgate/cli/scan/tools_agent.py +++ b/src/agents_shipgate/cli/scan/tools_agent.py @@ -2,6 +2,7 @@ import logging +from agents_shipgate.core.agent_bindings import resolve_agent_binding_graph from agents_shipgate.core.risk_hints import enrich_tools_with_risk_hints from agents_shipgate.core.semantic_assessment import attach_semantic_assessments from agents_shipgate.core.tool_identity import resolve_selectors_by_tool_id @@ -42,13 +43,24 @@ def _build_tools_and_agent( # Some adapters expose the same warnings through both LoadedToolSource # and the artifact bag; keep report warning output stable and unique. warnings = list(dict.fromkeys(warnings)) - tools = enrich_tools_with_risk_hints(manifest, tools) - declarations, tools = resolve_selectors_by_tool_id( - tools, + tool_catalog = enrich_tools_with_risk_hints(manifest, tools) + binding_graph, tool_catalog = resolve_agent_binding_graph( + manifest, + tool_catalog, + inputs.artifact_bag, + ) + reachable_ids = set(binding_graph.reachable_tool_ids) + declarations, tool_catalog = resolve_selectors_by_tool_id( + tool_catalog, manifest.action_surface.actions, manifest_path="/action_surface/actions", copy_tools=False, ) + declarations = { + tool_id: declaration + for tool_id, declaration in declarations.items() + if tool_id in reachable_ids + } for control, entries, path in ( ( "approval", @@ -66,15 +78,16 @@ def _build_tools_and_agent( "/policies/require_idempotency_for_tools", ), ): - resolved_controls, tools = resolve_selectors_by_tool_id( - tools, + resolved_controls, tool_catalog = resolve_selectors_by_tool_id( + tool_catalog, entries, manifest_path=path, copy_tools=False, ) - for tool in tools: + for tool in tool_catalog: if tool.id in resolved_controls: tool.resolved_controls = sorted(set(tool.resolved_controls) | {control}) + tools = [tool for tool in tool_catalog if tool.id in reachable_ids] tools = attach_semantic_assessments( tools, declarations, @@ -105,5 +118,10 @@ def _build_tools_and_agent( bound for loaded in inputs.loaded_sources for bound in loaded.toolkit_bounds ] return _ToolsAndAgent( - tools=tools, agent=agent, warnings=warnings, toolkit_bounds=toolkit_bounds + tools=tools, + tool_catalog=tool_catalog, + agent=agent, + binding_graph=binding_graph, + warnings=warnings, + toolkit_bounds=toolkit_bounds, ) diff --git a/src/agents_shipgate/core/agent_bindings.py b/src/agents_shipgate/core/agent_bindings.py new file mode 100644 index 00000000..aa009f1a --- /dev/null +++ b/src/agents_shipgate/core/agent_bindings.py @@ -0,0 +1,858 @@ +from __future__ import annotations + +import hashlib +import json +from collections import defaultdict, deque +from dataclasses import dataclass +from typing import Any + +from agents_shipgate.core.artifact_models import ( + CrewAiArtifacts, + GoogleAdkArtifacts, + LangChainArtifacts, + N8nArtifacts, +) +from agents_shipgate.core.artifacts import ArtifactBag +from agents_shipgate.core.domain import ( + BindingSemanticAssessment, + SemanticClaim, + SemanticIssue, + Tool, +) +from agents_shipgate.core.tool_identity import ToolSelectorIndex +from agents_shipgate.schemas.bindings import ( + AgentBindingGraphAssessment, + AgentBindingIssue, + AgentBindingNode, + AgentHandoffBindingEdge, + AgentToolBindingEdge, +) +from agents_shipgate.schemas.manifest import AgentsShipgateManifest + +_TOOL_EDGE_TYPES = { + "direct_tool", + "tool_node", + "toolset", + "workflow", +} +_HANDOFF_EDGE_TYPES = {"subagent", "handoff"} + + +@dataclass(frozen=True) +class _AgentObservation: + name: str + source_id: str | None + source_ref: str | None + source_pointer: str | None + complete: bool + + @property + def agent_id(self) -> str: + payload = json.dumps( + { + "name": self.name, + "source_id": self.source_id, + "source_ref": None if self.source_id else self.source_ref, + }, + sort_keys=True, + separators=(",", ":"), + ) + return f"agent_v1:{hashlib.sha256(payload.encode()).hexdigest()[:24]}" + + +@dataclass(frozen=True) +class _RawToolEdge: + agent_name: str + source_id: str | None + tool_name: str + edge_type: str + source: str + source_pointer: str | None + complete: bool + + +@dataclass(frozen=True) +class _RawHandoffEdge: + source_agent: str + target_agent: str + source_id: str | None + edge_type: str + source: str + source_pointer: str | None + complete: bool + + +def resolve_agent_binding_graph( + manifest: AgentsShipgateManifest, + tools: list[Tool], + artifacts: ArtifactBag, +) -> tuple[AgentBindingGraphAssessment, list[Tool]]: + """Resolve the exact static graph rooted at the configured agent. + + Catalog membership is deliberately not evidence. Only reviewed manifest + declarations and framework-specific structural observations create edges. + """ + + ( + agent_observations, + raw_tool_edges, + raw_handoffs, + partials, + invalid_annotations, + ) = _observations(tools, artifacts) + declarations = manifest.agent_bindings.declarations + for declaration in declarations: + if declaration.agent != "root": + agent_observations.append( + _AgentObservation(declaration.agent, None, "shipgate.yaml", None, True) + ) + for handoff in declaration.handoffs: + agent_observations.append( + _AgentObservation(handoff, None, "shipgate.yaml", None, True) + ) + + agents = _dedupe_agents(agent_observations) + root, root_issues = _select_root(manifest, agents, raw_handoffs) + if root is not None and all(agent.agent_id != root.agent_id for agent in agents): + agents.append(root) + issues: list[AgentBindingIssue] = list(root_issues) + issues.extend( + AgentBindingIssue( + kind="invalid_binding_annotation", + message=message, + source="framework_extraction", + ) + for message in sorted(invalid_annotations) + ) + selector_index = ToolSelectorIndex.build(tools) + by_agent_name = _agents_by_name(agents) + tool_edges: list[AgentToolBindingEdge] = [] + handoff_edges: list[AgentHandoffBindingEdge] = [] + + for raw in sorted( + raw_tool_edges, + key=lambda item: ( + item.agent_name, + item.source_id or "", + item.tool_name, + item.source, + item.source_pointer or "", + ), + ): + agent = _resolve_agent(raw.agent_name, raw.source_id, by_agent_name) + if agent is None: + issues.append( + AgentBindingIssue( + kind="unresolved_agent_binding", + message=f"Binding references unresolved agent {raw.agent_name!r}.", + source=raw.source, + source_pointer=raw.source_pointer, + ) + ) + continue + matches = [ + tool + for tool in tools + if tool.name == raw.tool_name + and ( + raw.source_id is None + or tool.source_id == raw.source_id + or tool.provider == raw.source_id + or tool.annotations.get("adk_agent_source_id") == raw.source_id + or tool.annotations.get("n8n_workflow_id") == raw.source_id + ) + ] + if len(matches) != 1: + issues.append( + AgentBindingIssue( + kind="unresolved_bound_tool", + message=( + f"Structural binding for {raw.tool_name!r} matched " + f"{len(matches)} canonical tools." + ), + agent_id=agent.agent_id, + source=raw.source, + source_pointer=raw.source_pointer, + ) + ) + continue + if raw.edge_type not in _TOOL_EDGE_TYPES: + issues.append( + AgentBindingIssue( + kind="invalid_binding_annotation", + message=f"Unknown tool binding edge type {raw.edge_type!r}.", + agent_id=agent.agent_id, + tool_id=matches[0].id, + source=raw.source, + source_pointer=raw.source_pointer, + ) + ) + continue + tool_edges.append( + AgentToolBindingEdge( + agent_id=agent.agent_id, + tool_id=matches[0].id, + edge_type=raw.edge_type, + confidence="high" if raw.complete else "medium", + provenance_kind="ast_extraction", + source=raw.source, + source_pointer=raw.source_pointer, + complete=raw.complete, + ) + ) + + for raw in raw_handoffs: + source_agent = _resolve_agent(raw.source_agent, raw.source_id, by_agent_name) + target_agent = _resolve_agent(raw.target_agent, raw.source_id, by_agent_name) + if source_agent is None or target_agent is None: + issues.append( + AgentBindingIssue( + kind="incomplete_handoff_graph", + message=( + f"Handoff {raw.source_agent!r} -> {raw.target_agent!r} " + "could not be resolved exactly." + ), + source=raw.source, + source_pointer=raw.source_pointer, + ) + ) + continue + if raw.edge_type not in _HANDOFF_EDGE_TYPES: + issues.append( + AgentBindingIssue( + kind="invalid_binding_annotation", + message=f"Unknown handoff edge type {raw.edge_type!r}.", + agent_id=source_agent.agent_id, + source=raw.source, + source_pointer=raw.source_pointer, + ) + ) + continue + handoff_edges.append( + AgentHandoffBindingEdge( + source_agent_id=source_agent.agent_id, + target_agent_id=target_agent.agent_id, + edge_type=raw.edge_type, + confidence="high" if raw.complete else "medium", + provenance_kind="ast_extraction", + source=raw.source, + source_pointer=raw.source_pointer, + complete=raw.complete, + ) + ) + + structural_by_agent: dict[str, set[str]] = defaultdict(set) + structural_complete: dict[str, bool] = defaultdict(lambda: True) + for edge in tool_edges: + structural_by_agent[edge.agent_id].add(edge.tool_id) + structural_complete[edge.agent_id] &= edge.complete + structural_handoffs: dict[str, set[str]] = defaultdict(set) + structural_handoffs_complete: dict[str, bool] = defaultdict(lambda: True) + for edge in handoff_edges: + structural_handoffs[edge.source_agent_id].add(edge.target_agent_id) + structural_handoffs_complete[edge.source_agent_id] &= edge.complete + + for index, declaration in enumerate(declarations): + agent = root if declaration.agent == "root" else _resolve_agent( + declaration.agent, None, by_agent_name + ) + pointer = f"/agent_bindings/declarations/{index}" + if agent is None: + issues.append( + AgentBindingIssue( + kind="unresolved_agent_binding", + message=f"Declaration references unresolved agent {declaration.agent!r}.", + source="shipgate.yaml", + source_pointer=f"{pointer}/agent", + ) + ) + continue + declared_ids: set[str] = set() + for selector_index_value, selector in enumerate(declaration.tools): + result = selector_index.resolve(selector) + if not result.resolved: + issues.append( + AgentBindingIssue( + kind="unresolved_bound_tool", + message=result.message or "Binding selector did not resolve exactly.", + agent_id=agent.agent_id, + source="shipgate.yaml", + source_pointer=f"{pointer}/tools/{selector_index_value}", + ) + ) + continue + tool = result.matches[0] + declared_ids.add(tool.id) + tool_edges.append( + AgentToolBindingEdge( + agent_id=agent.agent_id, + tool_id=tool.id, + edge_type="direct_tool", + confidence="high", + provenance_kind="static_declaration", + source="agent_binding_declaration", + source_pointer=f"{pointer}/tools/{selector_index_value}", + complete=True, + ) + ) + structural_ids = structural_by_agent.get(agent.agent_id, set()) + if structural_ids and structural_complete[agent.agent_id] and declared_ids != structural_ids: + issues.append( + AgentBindingIssue( + kind="conflicting_binding_evidence", + message=( + f"Closed-world declaration for {declaration.agent!r} does not " + "match the complete structural tool set." + ), + agent_id=agent.agent_id, + source="shipgate.yaml", + source_pointer=pointer, + ) + ) + declared_handoff_ids: set[str] = set() + for handoff_index, target_name in enumerate(declaration.handoffs): + target = _resolve_agent(target_name, None, by_agent_name) + if target is None: + issues.append( + AgentBindingIssue( + kind="unresolved_agent_binding", + message=f"Declaration references unresolved handoff {target_name!r}.", + agent_id=agent.agent_id, + source="shipgate.yaml", + source_pointer=f"{pointer}/handoffs/{handoff_index}", + ) + ) + continue + declared_handoff_ids.add(target.agent_id) + handoff_edges.append( + AgentHandoffBindingEdge( + source_agent_id=agent.agent_id, + target_agent_id=target.agent_id, + edge_type="handoff", + confidence="high", + provenance_kind="static_declaration", + source="agent_binding_declaration", + source_pointer=f"{pointer}/handoffs/{handoff_index}", + complete=True, + ) + ) + structural_target_ids = structural_handoffs.get(agent.agent_id, set()) + if ( + structural_target_ids + and structural_handoffs_complete[agent.agent_id] + and declared_handoff_ids != structural_target_ids + ): + issues.append( + AgentBindingIssue( + kind="conflicting_binding_evidence", + message=( + f"Closed-world declaration for {declaration.agent!r} does not " + "match the complete structural handoff set." + ), + agent_id=agent.agent_id, + source="shipgate.yaml", + source_pointer=pointer, + ) + ) + + if partials: + for partial in sorted(partials): + issues.append( + AgentBindingIssue( + kind="partial_binding_evidence", + message=partial, + source="framework_extraction", + ) + ) + + tool_edges = _dedupe_tool_edges(tool_edges) + handoff_edges = _dedupe_handoff_edges(handoff_edges) + reachable_agents, paths = _reachable_agents(root, handoff_edges) + reachable_ids = sorted( + { + edge.tool_id + for edge in tool_edges + if edge.agent_id in reachable_agents and edge.complete + } + ) + possible_ids = sorted( + { + edge.tool_id + for edge in tool_edges + if edge.agent_id in reachable_agents and not edge.complete + } + - set(reachable_ids) + ) + unbound_ids = sorted(set(selector_index.by_id) - set(reachable_ids) - set(possible_ids)) + + if root is not None and not declarations and not tool_edges and tools: + issues.append( + AgentBindingIssue( + kind="missing_binding_evidence", + message=( + "Tools were extracted into the catalog, but no static edge binds " + "them to the configured root agent." + ), + agent_id=root.agent_id, + ) + ) + if root is None and tools and not issues: + issues.append( + AgentBindingIssue( + kind="ambiguous_root_agent", + message="The root agent could not be identified unambiguously.", + ) + ) + + conflict = any(issue.kind == "conflicting_binding_evidence" for issue in issues) + partial = any( + issue.kind + in { + "partial_binding_evidence", + "incomplete_handoff_graph", + "unresolved_agent_binding", + "unresolved_bound_tool", + } + for issue in issues + ) + declared = bool(declarations) and not conflict and not partial and root is not None + structural = bool(tool_edges or (root is not None and not tools)) and not issues + status = ( + "conflicting" + if conflict + else "partial" + if partial + else "declared" + if declared + else "structural" + if structural + else "unknown" + ) + pass_eligible = root is not None and not issues and status in {"declared", "structural"} + + graph = AgentBindingGraphAssessment( + root_agent_id=root.agent_id if root else None, + status=status, + agents=sorted(agents, key=lambda item: item.agent_id), + tool_edges=tool_edges, + handoff_edges=handoff_edges, + reachable_tool_ids=reachable_ids, + possible_tool_ids=possible_ids, + unbound_tool_ids=unbound_ids, + issues=sorted( + issues, + key=lambda item: ( + item.kind, + item.agent_id or "", + item.tool_id or "", + item.source or "", + item.source_pointer or "", + item.message, + ), + ), + pass_eligible=pass_eligible, + ) + _attach_binding_assessments(tools, graph, paths) + return graph, tools + + +def _observations( + tools: list[Tool], artifacts: ArtifactBag +) -> tuple[ + list[_AgentObservation], + list[_RawToolEdge], + list[_RawHandoffEdge], + set[str], + set[str], +]: + agents: list[_AgentObservation] = [] + edges: list[_RawToolEdge] = [] + handoffs: list[_RawHandoffEdge] = [] + partials: set[str] = set() + invalid_annotations: set[str] = set() + + for tool in tools: + for reason in _string_list(tool.annotations.get("binding_surface_partial")): + partials.add(reason) + raw_bindings = tool.annotations.get("agent_bindings") + if raw_bindings is not None and not isinstance(raw_bindings, list): + invalid_annotations.add( + f"Tool {tool.name!r} has a non-list structural binding annotation." + ) + if isinstance(raw_bindings, list): + for raw in raw_bindings: + if not isinstance(raw, dict) or not isinstance(raw.get("agent"), str): + invalid_annotations.add( + f"Tool {tool.name!r} has a malformed structural binding annotation." + ) + continue + source_id = raw.get("source_id") if isinstance(raw.get("source_id"), str) else tool.source_id + source = str(raw.get("source") or tool.source_ref or tool.source_type) + pointer = raw.get("source_pointer") if isinstance(raw.get("source_pointer"), str) else tool.source_location + raw_complete = raw.get("complete", True) + if type(raw_complete) is not bool: + invalid_annotations.add( + f"Tool {tool.name!r} has a non-boolean binding completeness annotation." + ) + continue + complete = raw_complete + agents.append(_AgentObservation(raw["agent"], source_id, source, pointer, complete)) + edges.append( + _RawToolEdge( + raw["agent"], + source_id, + tool.name, + str(raw.get("edge_type") or "direct_tool"), + source, + pointer, + complete, + ) + ) + raw_handoffs = tool.annotations.get("agent_handoffs") + if raw_handoffs is not None and not isinstance(raw_handoffs, list): + invalid_annotations.add( + f"Tool {tool.name!r} has a non-list handoff annotation." + ) + if isinstance(raw_handoffs, list): + for raw in raw_handoffs: + if not isinstance(raw, dict): + invalid_annotations.add( + f"Tool {tool.name!r} has a malformed handoff annotation." + ) + continue + source_agent = raw.get("source_agent") + target_agent = raw.get("target_agent") + if not isinstance(source_agent, str) or not isinstance(target_agent, str): + invalid_annotations.add( + f"Tool {tool.name!r} has a handoff without exact agent names." + ) + continue + raw_complete = raw.get("complete", True) + if type(raw_complete) is not bool: + invalid_annotations.add( + f"Tool {tool.name!r} has a non-boolean handoff completeness annotation." + ) + continue + handoffs.append( + _RawHandoffEdge( + source_agent, + target_agent, + _str(raw.get("source_id")) or tool.source_id, + str(raw.get("edge_type") or "handoff"), + str(raw.get("source") or tool.source_ref or tool.source_type), + _str(raw.get("source_pointer")) or tool.source_location, + raw_complete, + ) + ) + adk_agent = tool.annotations.get("adk_agent_name") + if isinstance(adk_agent, str): + adk_source_id = _str(tool.annotations.get("adk_agent_source_id")) or tool.source_id + agents.append(_AgentObservation(adk_agent, adk_source_id, tool.source_ref, tool.source_location, True)) + edges.append(_RawToolEdge(adk_agent, adk_source_id, tool.name, "direct_tool", tool.source_ref or "google_adk", tool.source_location, True)) + if tool.source_type == "codex_plugin_mcp_inventory": + plugin = _str(tool.annotations.get("codex_plugin")) + server = _str(tool.annotations.get("codex_plugin_mcp_server")) + if plugin and server: + agent_name = f"{plugin}/{server}" + agents.append( + _AgentObservation( + agent_name, + tool.source_id, + tool.source_ref, + tool.source_location, + True, + ) + ) + edges.append( + _RawToolEdge( + agent_name, + tool.source_id, + tool.name, + "direct_tool", + tool.source_ref or "codex_plugin_mcp_inventory", + tool.source_location, + True, + ) + ) + + langchain = artifacts.get("langchain", LangChainArtifacts) + if langchain: + for record in langchain.agent_bindings: + name = str(record.get("agent") or "root") + source_ref = _str(record.get("source_ref")) + agents.append(_AgentObservation(name, _str(record.get("source_id")), source_ref, _line_pointer(record), True)) + for tool_name in _string_list(record.get("tools")): + edges.append(_RawToolEdge(name, _str(record.get("source_id")), tool_name, "tool_node" if record.get("kind") == "tool_node" else "direct_tool", source_ref or "langchain", _line_pointer(record), True)) + for record in langchain.tool_nodes: + name = str(record.get("agent") or record.get("name") or "root") + source_ref = _str(record.get("source_ref")) + agents.append(_AgentObservation(name, _str(record.get("source_id")), source_ref, _line_pointer(record), True)) + for tool_name in _string_list(record.get("tools")): + edges.append(_RawToolEdge(name, _str(record.get("source_id")), tool_name, "tool_node", source_ref or "langchain", _line_pointer(record), True)) + partials.update(str(item.get("reason")) for item in langchain.dynamic_tool_surfaces if item.get("reason")) + + crewai = artifacts.get("crewai", CrewAiArtifacts) + if crewai: + for record in crewai.agents: + name = str(record.get("name") or "root") + source_ref = _str(record.get("source_ref")) + agents.append(_AgentObservation(name, _str(record.get("source_id")), source_ref, _line_pointer(record), True)) + for tool_name in _string_list(record.get("tools")): + edges.append(_RawToolEdge(name, _str(record.get("source_id")), tool_name, "direct_tool", source_ref or "crewai", _line_pointer(record), True)) + partials.update(str(item.get("reason")) for item in crewai.dynamic_tool_surfaces if item.get("reason")) + for record in crewai.crews: + crew_name = str(record.get("name") or "crew") + source_id = _str(record.get("source_id")) + source_ref = _str(record.get("source_ref")) + agents.append(_AgentObservation(crew_name, source_id, source_ref, _line_pointer(record), True)) + for member in _string_list(record.get("agents")): + handoffs.append( + _RawHandoffEdge( + crew_name, + member, + source_id, + "subagent", + source_ref or "crewai", + _line_pointer(record), + True, + ) + ) + + adk = artifacts.get("google_adk", GoogleAdkArtifacts) + if adk: + for record in adk.agents: + name = str(record.get("name") or "root") + agents.append(_AgentObservation(name, _str(record.get("source_id")), _str(record.get("source_ref")), _str(record.get("source_ref")), True)) + for record in adk.sub_agents: + source_name = _str(record.get("agent_name")) + for target in _string_list(record.get("sub_agents")): + if source_name: + agents.append(_AgentObservation(target, _str(record.get("source_id")), _str(record.get("source_ref")), _str(record.get("source_ref")), True)) + handoffs.append(_RawHandoffEdge(source_name, target, _str(record.get("source_id")), "subagent", _str(record.get("source_ref")) or "google_adk", _str(record.get("source_ref")), True)) + if record.get("sub_agent_count") and not record.get("sub_agents"): + partials.add(f"Google ADK agent {source_name!r} has sub-agents that were not statically named.") + for toolset in adk.toolsets: + if toolset.dynamic or not toolset.resolved: + partials.add(f"Google ADK toolset {toolset.name or toolset.kind!r} is not statically enumerable.") + + n8n = artifacts.get("n8n", N8nArtifacts) + if n8n: + for record in n8n.ai_agents: + agents.append(_AgentObservation(str(record.get("name") or record.get("node_id") or "root"), _str(record.get("workflow_id")), _str(record.get("source_ref")), _str(record.get("source_pointer")), True)) + partials.update(str(item.get("reason")) for item in n8n.dynamic_tool_surfaces if item.get("reason")) + + return agents, edges, handoffs, partials, invalid_annotations + + +def _select_root( + manifest: AgentsShipgateManifest, + agents: list[AgentBindingNode], + raw_handoffs: list[_RawHandoffEdge], +) -> tuple[AgentBindingNode | None, list[AgentBindingIssue]]: + root_config = manifest.agent_bindings.root + candidates = agents + if root_config is not None: + candidates = [agent for agent in agents if agent.name == root_config.object] + if root_config.source_id: + candidates = [agent for agent in candidates if agent.source_id == root_config.source_id] + elif manifest.agent.sdk and manifest.agent.sdk.object: + candidates = [agent for agent in agents if agent.name == manifest.agent.sdk.object] + elif len(manifest.agent_bindings.declarations) == 1: + declared = manifest.agent_bindings.declarations[0].agent + if declared == "root": + if len(agents) == 1: + return agents[0], [] + synthetic = AgentBindingNode( + agent_id=_AgentObservation("root", None, "shipgate.yaml", None, True).agent_id, + name="root", + source_ref="shipgate.yaml", + ) + return synthetic, [] + candidates = [agent for agent in agents if agent.name == declared] + else: + target_names = {edge.target_agent for edge in raw_handoffs} + top_level = [agent for agent in agents if agent.name not in target_names] + if len(top_level) == 1: + candidates = top_level + if len(candidates) == 1: + return candidates[0], [] + message = ( + "No root agent matched the configured selector." + if not candidates + else f"Root selector matched {len(candidates)} agents." + ) + return None, [AgentBindingIssue(kind="ambiguous_root_agent", message=message, source="shipgate.yaml")] + + +def _dedupe_agents(observations: list[_AgentObservation]) -> list[AgentBindingNode]: + by_key: dict[tuple[str, str | None, str | None], _AgentObservation] = {} + for observation in observations: + by_key[ + ( + observation.name, + observation.source_id, + None if observation.source_id else observation.source_ref, + ) + ] = observation + return [ + AgentBindingNode( + agent_id=observation.agent_id, + name=observation.name, + source_id=observation.source_id, + source_ref=observation.source_ref, + source_pointer=observation.source_pointer, + ) + for observation in by_key.values() + ] + + +def _agents_by_name(agents: list[AgentBindingNode]) -> dict[str, list[AgentBindingNode]]: + result: dict[str, list[AgentBindingNode]] = defaultdict(list) + for agent in agents: + result[agent.name].append(agent) + return result + + +def _resolve_agent(name: str, source_id: str | None, by_name: dict[str, list[AgentBindingNode]]) -> AgentBindingNode | None: + candidates = by_name.get(name, []) + if source_id is not None: + candidates = [agent for agent in candidates if agent.source_id == source_id] + return candidates[0] if len(candidates) == 1 else None + + +def _reachable_agents( + root: AgentBindingNode | None, + handoffs: list[AgentHandoffBindingEdge], +) -> tuple[set[str], dict[str, list[str]]]: + if root is None: + return set(), {} + outgoing: dict[str, list[str]] = defaultdict(list) + for edge in handoffs: + if edge.complete: + outgoing[edge.source_agent_id].append(edge.target_agent_id) + reachable = {root.agent_id} + paths = {root.agent_id: [root.agent_id]} + queue = deque([root.agent_id]) + while queue: + current = queue.popleft() + for target in sorted(outgoing.get(current, [])): + if target in reachable: + continue + reachable.add(target) + paths[target] = [*paths[current], target] + queue.append(target) + return reachable, paths + + +def _attach_binding_assessments( + tools: list[Tool], + graph: AgentBindingGraphAssessment, + agent_paths: dict[str, list[str]], +) -> None: + edges_by_tool: dict[str, list[AgentToolBindingEdge]] = defaultdict(list) + for edge in graph.tool_edges: + if edge.tool_id in graph.reachable_tool_ids: + edges_by_tool[edge.tool_id].append(edge) + graph_issue_by_tool: dict[str, list[AgentBindingIssue]] = defaultdict(list) + graph_issue_by_agent: dict[str, list[AgentBindingIssue]] = defaultdict(list) + for issue in graph.issues: + if issue.tool_id: + graph_issue_by_tool[issue.tool_id].append(issue) + if issue.agent_id: + graph_issue_by_agent[issue.agent_id].append(issue) + for tool in tools: + edges = sorted( + edges_by_tool.get(tool.id, []), + key=lambda edge: (edge.agent_id, edge.source, edge.source_pointer or ""), + ) + if edges: + edge = edges[0] + claims = [ + SemanticClaim( + dimension="binding", + value=f"{item.agent_id}->{tool.id}", + confidence=item.confidence, + provenance_kind=item.provenance_kind, + source=item.source, + source_pointer=item.source_pointer, + evidence={"edge_type": item.edge_type, "complete": item.complete}, + ) + for item in edges + ] + issues = [ + SemanticIssue( + kind=issue.kind, + dimension="binding", + message=issue.message, + source=issue.source, + source_pointer=issue.source_pointer, + ) + for issue in [ + *graph_issue_by_tool.get(tool.id, []), + *graph_issue_by_agent.get(edge.agent_id, []), + ] + ] + status = "declared" if any(item.provenance_kind == "static_declaration" for item in edges) else "structural" + tool.binding_assessment = BindingSemanticAssessment( + status=status, + confidence="high", + root_agent_id=graph.root_agent_id, + reachable_path=[*agent_paths.get(edge.agent_id, [edge.agent_id]), tool.id], + claims=claims, + issues=issues, + pass_eligible=not issues and all(item.complete for item in edges), + ) + else: + tool.binding_assessment = BindingSemanticAssessment( + status="unknown", + confidence="low", + root_agent_id=graph.root_agent_id, + pass_eligible=False, + ) + + +def _dedupe_tool_edges(edges: list[AgentToolBindingEdge]) -> list[AgentToolBindingEdge]: + by_key = { + ( + edge.agent_id, + edge.tool_id, + edge.edge_type, + edge.provenance_kind, + edge.source, + edge.source_pointer, + ): edge + for edge in edges + } + return [by_key[key] for key in sorted(by_key)] + + +def _dedupe_handoff_edges(edges: list[AgentHandoffBindingEdge]) -> list[AgentHandoffBindingEdge]: + by_key = { + ( + edge.source_agent_id, + edge.target_agent_id, + edge.edge_type, + edge.provenance_kind, + edge.source, + edge.source_pointer, + ): edge + for edge in edges + } + return [by_key[key] for key in sorted(by_key)] + + +def _string_list(value: Any) -> list[str]: + return [item for item in value if isinstance(item, str)] if isinstance(value, list) else [] + + +def _str(value: Any) -> str | None: + return value if isinstance(value, str) and value else None + + +def _line_pointer(record: dict[str, Any]) -> str | None: + source = _str(record.get("source_ref")) + line = record.get("line") + return f"{source}:{line}" if source and isinstance(line, int) else source + + +__all__ = ["resolve_agent_binding_graph"] diff --git a/src/agents_shipgate/core/capabilities.py b/src/agents_shipgate/core/capabilities.py index b47b66a5..451e57f2 100644 --- a/src/agents_shipgate/core/capabilities.py +++ b/src/agents_shipgate/core/capabilities.py @@ -58,6 +58,11 @@ def capability_fact_from_action( semantic_evidence = _semantic_evidence(action) hashes = CapabilityHashes( identity_hash=_capability_stable_hash(identity.model_dump(mode="json")), + binding_hash=_capability_stable_hash( + semantic_evidence.binding.model_dump(mode="json") + if semantic_evidence is not None + else {"status": "unknown"} + ), effect_hash=_capability_stable_hash(effect.model_dump(mode="json")), authority_hash=_capability_stable_hash(authority.model_dump(mode="json")), control_hash=_capability_stable_hash(controls.model_dump(mode="json")), @@ -214,6 +219,11 @@ def capability_fact_from_action_fact(action: ActionFact) -> CapabilityFactV1: semantic_evidence = action.semantic_assessment hashes = CapabilityHashes( identity_hash=_capability_stable_hash(identity.model_dump(mode="json")), + binding_hash=_capability_stable_hash( + semantic_evidence.binding.model_dump(mode="json") + if semantic_evidence is not None + else {"status": "unknown"} + ), effect_hash=_capability_stable_hash(effect.model_dump(mode="json")), authority_hash=_capability_stable_hash(authority.model_dump(mode="json")), control_hash=_capability_stable_hash(controls.model_dump(mode="json")), diff --git a/src/agents_shipgate/core/capability_delta.py b/src/agents_shipgate/core/capability_delta.py index a548c868..fda5c2f4 100644 --- a/src/agents_shipgate/core/capability_delta.py +++ b/src/agents_shipgate/core/capability_delta.py @@ -17,6 +17,7 @@ SEMANTIC_CAPABILITY_HASH_FIELDS: tuple[CapabilityHashName, ...] = ( "identity_hash", + "binding_hash", "effect_hash", "authority_hash", "control_hash", @@ -668,6 +669,7 @@ def _semantic_direction( def _hash_name_explained(name: CapabilityHashName, fields: set[str]) -> bool: prefixes = { "identity_hash": ("identity.",), + "binding_hash": ("semantic_assessment.binding.",), "effect_hash": ("effect.",), "authority_hash": ("authority.",), "control_hash": ("controls.",), diff --git a/src/agents_shipgate/core/capability_lock.py b/src/agents_shipgate/core/capability_lock.py index 3f0ecf0c..4a348a90 100644 --- a/src/agents_shipgate/core/capability_lock.py +++ b/src/agents_shipgate/core/capability_lock.py @@ -176,6 +176,7 @@ def _lock_hashes( { "id": fact.id, "identity_hash": fact.hashes.identity_hash, + "binding_hash": fact.hashes.binding_hash, "effect_hash": fact.hashes.effect_hash, "authority_hash": fact.hashes.authority_hash, "control_hash": fact.hashes.control_hash, @@ -228,10 +229,23 @@ def _lock_ref(lock: CapabilityLockFileV1, *, path: Path | None) -> CapabilityLoc def _normalize_capability_lock_payload(payload: dict[str, Any]) -> dict[str, Any]: version = payload.get("capability_lock_schema_version") - if version in {"0.1", "0.2", "0.3"}: + if version in {"0.1", "0.2", "0.3", "0.4"}: normalized = dict(payload) normalized["capability_lock_schema_version"] = CAPABILITY_LOCK_SCHEMA_VERSION normalized["experimental"] = False + normalized["capabilities"] = [ + { + **fact, + "hashes": { + **fact.get("hashes", {}), + "binding_hash": fact.get("hashes", {}).get( + "binding_hash", "legacy_binding_unknown" + ), + }, + } + for fact in payload.get("capabilities", []) + if isinstance(fact, dict) + ] return normalized return payload @@ -240,6 +254,7 @@ def _normalize_capability_lock_payload(payload: dict[str, Any]) -> dict[str, Any "0.1": "0.1", "0.2": "0.1", "0.3": "0.2", + "0.4": "0.3", CAPABILITY_LOCK_SCHEMA_VERSION: CAPABILITY_STANDARD_VERSION, } diff --git a/src/agents_shipgate/core/context.py b/src/agents_shipgate/core/context.py index 66a35f8b..de0d8df3 100644 --- a/src/agents_shipgate/core/context.py +++ b/src/agents_shipgate/core/context.py @@ -24,6 +24,7 @@ ) from agents_shipgate.core.lenses.tool_surface import ToolSurfaceDiffReference from agents_shipgate.inputs.common import PositionIndex +from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment from agents_shipgate.schemas.capabilities import CapabilityFactV1 from agents_shipgate.schemas.manifest import AgentsShipgateManifest from agents_shipgate.schemas.report import CapabilityRuntimeEvidence @@ -52,6 +53,8 @@ class ScanContext: agent: Agent tools: list[Tool] config_path: Path + tool_catalog: list[Tool] = field(default_factory=list) + binding_graph: AgentBindingGraphAssessment | None = None framework_artifacts: ArtifactBag = field(default_factory=ArtifactBag) action_surface_facts: ActionSurfaceFacts = field(default_factory=ActionSurfaceFacts) capability_facts: list[CapabilityFactV1] = field(default_factory=list) diff --git a/src/agents_shipgate/core/domain.py b/src/agents_shipgate/core/domain.py index 75f2ef57..2c9017e7 100644 --- a/src/agents_shipgate/core/domain.py +++ b/src/agents_shipgate/core/domain.py @@ -8,7 +8,7 @@ from agents_shipgate.schemas.common import Confidence, ProvenanceKind, parse_confidence from agents_shipgate.schemas.surfaces import ActionEffect -SemanticDimension = Literal["identity", "effect", "authority"] +SemanticDimension = Literal["identity", "binding", "effect", "authority"] IdentityEvidenceStatus = Literal[ "declared", "structural", @@ -47,6 +47,14 @@ "partial_authority_evidence", "conflicting_authority_evidence", "invalid_semantic_annotation", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation", ] @@ -113,6 +121,30 @@ class ToolIdentityAssessment(BaseModel): pass_eligible: bool +class BindingSemanticAssessment(BaseModel): + """Static proof that a tool is reachable from the configured root agent.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + status: Literal["declared", "structural", "partial", "unknown", "conflicting"] + confidence: Confidence + root_agent_id: str | None = None + reachable_path: list[str] = Field(default_factory=list) + claims: list[SemanticClaim] = Field(default_factory=list) + issues: list[SemanticIssue] = Field(default_factory=list) + pass_eligible: bool = False + + +def _legacy_direct_binding() -> BindingSemanticAssessment: + return BindingSemanticAssessment( + status="structural", + confidence="high", + root_agent_id="legacy_direct", + reachable_path=["legacy_direct"], + pass_eligible=True, + ) + + def _legacy_direct_identity() -> ToolIdentityAssessment: return ToolIdentityAssessment( tool_id="legacy_direct", @@ -131,6 +163,7 @@ class ToolSemanticAssessment(BaseModel): conservative_effect: ActionEffect identity: ToolIdentityAssessment = Field(default_factory=_legacy_direct_identity) + binding: BindingSemanticAssessment = Field(default_factory=_legacy_direct_binding) effect: EffectSemanticAssessment authority: AuthoritySemanticAssessment pass_eligible: bool @@ -234,6 +267,10 @@ class Tool(BaseModel): default=None, exclude=True, ) + binding_assessment: BindingSemanticAssessment | None = Field( + default=None, + exclude=True, + ) resolved_controls: list[str] = Field(default_factory=list, exclude=True) @model_validator(mode="after") diff --git a/src/agents_shipgate/core/findings/report_builder.py b/src/agents_shipgate/core/findings/report_builder.py index 4c1ad818..e5adc0e2 100644 --- a/src/agents_shipgate/core/findings/report_builder.py +++ b/src/agents_shipgate/core/findings/report_builder.py @@ -2,6 +2,7 @@ from agents_shipgate.ci.release_decision import build_release_decision from agents_shipgate.core.domain import Tool +from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment, BindingSurfaceDiff from agents_shipgate.schemas.codex_plugin import CodexPluginSurface from agents_shipgate.schemas.common import Severity from agents_shipgate.schemas.manifest import AgentsShipgateManifest @@ -39,6 +40,9 @@ def build_report( agent: dict[str, object], environment: dict[str, object], tools: list[Tool], + tool_catalog: list[Tool] | None = None, + binding_surface_facts: AgentBindingGraphAssessment | None = None, + binding_surface_diff: BindingSurfaceDiff | None = None, findings: list[Finding], generated_reports: dict[str, str], ci_mode: str, @@ -75,6 +79,15 @@ def build_report( tool_surface_diff=tool_surface_diff or ToolSurfaceDiff(), action_surface_facts=action_surface_facts or ActionSurfaceFacts(), action_surface_diff=action_surface_diff or ActionSurfaceDiff(), + binding_surface_facts=( + binding_surface_facts + or AgentBindingGraphAssessment( + root_agent_id="legacy_direct", + status="structural", + pass_eligible=True, + ) + ), + binding_surface_diff=binding_surface_diff or BindingSurfaceDiff(), capability_runtime_evidence=( capability_runtime_evidence or CapabilityRuntimeEvidence() ), @@ -90,6 +103,7 @@ def build_report( loaded_plugins=loaded_plugins or [], loaded_adapters=loaded_adapters or [], tool_inventory=tool_inventory(tools), + tool_catalog=tool_inventory(tool_catalog if tool_catalog is not None else tools), source_warnings=source_warnings or [], # v0.17 (M1): policy audit envelope. Always present on emitted # scans (empty when no overrides applied) so consumers can read diff --git a/src/agents_shipgate/core/findings/summaries.py b/src/agents_shipgate/core/findings/summaries.py index f0c59bc4..360eb04c 100644 --- a/src/agents_shipgate/core/findings/summaries.py +++ b/src/agents_shipgate/core/findings/summaries.py @@ -91,6 +91,11 @@ def tool_inventory(tools: list[Tool]) -> list[dict[str, object]]: if tool.semantic_assessment is not None else None ), + "binding_assessment": ( + tool.binding_assessment.model_dump(mode="json") + if tool.binding_assessment is not None + else None + ), } for tool in sorted(tools, key=lambda item: item.id) ] diff --git a/src/agents_shipgate/core/lenses/tool_surface.py b/src/agents_shipgate/core/lenses/tool_surface.py index dc94f394..28070441 100644 --- a/src/agents_shipgate/core/lenses/tool_surface.py +++ b/src/agents_shipgate/core/lenses/tool_surface.py @@ -20,6 +20,7 @@ from agents_shipgate.core.tool_identity import ToolSelectorIndex from agents_shipgate.core.toolkit_scope import toolkit_bound_facts from agents_shipgate.schemas.baseline import BaselineFile +from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment from agents_shipgate.schemas.capability_change import EffectivePolicy from agents_shipgate.schemas.manifest import ( AgentsShipgateManifest, @@ -75,6 +76,7 @@ class ToolSurfaceDiffReference: # the reference is a baseline (no policy snapshot) or a pre-v0.22 # report that predates the block. effective_policy: EffectivePolicy | None = None + binding_facts: AgentBindingGraphAssessment | None = None def build_tool_surface_facts( @@ -926,18 +928,28 @@ def _reference_from_report_payload( # Present on v0.22+ base reports; None for older bases (the # weakening checks degrade safely when it is absent). effective_policy=report.effective_policy, + binding_facts=( + report.binding_surface_facts + if "binding_surface_facts" in payload + and _schema_version_at_least(report.report_schema_version, "0.31") + else None + ), ) def _report_schema_precedes_semantic_diff(value: Any) -> bool: + return not _schema_version_at_least(value, _SEMANTIC_DIFF_REPORT_SCHEMA_VERSION) + + +def _schema_version_at_least(value: Any, minimum_value: str) -> bool: if not isinstance(value, str): - return True + return False try: current = tuple(int(part) for part in value.split(".")) - minimum = tuple(int(part) for part in _SEMANTIC_DIFF_REPORT_SCHEMA_VERSION.split(".")) + minimum = tuple(int(part) for part in minimum_value.split(".")) except ValueError: - return True - return current < minimum + return False + return current >= minimum def _reference_from_baseline_payload( diff --git a/src/agents_shipgate/core/semantic_assessment.py b/src/agents_shipgate/core/semantic_assessment.py index 61cfe1ea..f7d3e288 100644 --- a/src/agents_shipgate/core/semantic_assessment.py +++ b/src/agents_shipgate/core/semantic_assessment.py @@ -7,6 +7,7 @@ from agents_shipgate.core.domain import ( AuthorityMode, AuthoritySemanticAssessment, + BindingSemanticAssessment, EffectSemanticAssessment, Scope, SemanticClaim, @@ -101,10 +102,12 @@ def assess_tool_semantics( effect, conservative_effect = _assess_effect(tool, declaration) authority = _assess_authority(tool, declaration) identity = tool.identity_assessment or _compat_identity_assessment(tool) + binding = tool.binding_assessment or _compat_binding_assessment(tool) surface_complete = _surface_is_complete(tool) extraction_complete = tool.extraction_confidence == "high" pass_eligible = ( identity.pass_eligible + and binding.pass_eligible and surface_complete and extraction_complete and effect.status in {"declared", "structural"} @@ -117,12 +120,38 @@ def assess_tool_semantics( return ToolSemanticAssessment( conservative_effect=conservative_effect, identity=identity, + binding=binding, effect=effect, authority=authority, pass_eligible=pass_eligible, ) +def _compat_binding_assessment(tool: Tool) -> BindingSemanticAssessment: + """Compatibility for focused unit callers outside the scan pipeline. + + Production scans always attach graph-derived binding evidence before this + resolver runs. Direct semantic unit tests model one already-selected tool. + """ + + claim = SemanticClaim( + dimension="binding", + value=f"legacy_direct->{tool.id}", + confidence="high", + provenance_kind="static_declaration", + source="compat_direct_binding", + source_pointer=tool.source_pointer or tool.source_ref, + ) + return BindingSemanticAssessment( + status="structural", + confidence="high", + root_agent_id="legacy_direct", + reachable_path=["legacy_direct", tool.id], + claims=[claim], + pass_eligible=True, + ) + + def attach_semantic_assessments( tools: list[Tool], declarations: Mapping[str, ActionDeclarationConfig] | None = None, diff --git a/src/agents_shipgate/core/semantic_consistency.py b/src/agents_shipgate/core/semantic_consistency.py index a9764bac..fbfbf938 100644 --- a/src/agents_shipgate/core/semantic_consistency.py +++ b/src/agents_shipgate/core/semantic_consistency.py @@ -35,6 +35,24 @@ def validate_semantic_consistency( raise SemanticConsistencyError(f"duplicate assessed tool identity for {tool.name!r}") assessments[key] = tool.semantic_assessment + assessed_ids = {key[0] for key in assessments} + graph = report.binding_surface_facts + if assessed_ids != set(graph.reachable_tool_ids): + raise SemanticConsistencyError( + "binding graph reachable tools do not match the assessed tool surface" + ) + inventory_ids = {str(item.get("tool_id")) for item in report.tool_inventory} + if inventory_ids != assessed_ids: + raise SemanticConsistencyError("tool_inventory is not the reachable tool surface") + catalog_ids = {str(item.get("tool_id")) for item in report.tool_catalog} + graph_catalog_ids = ( + set(graph.reachable_tool_ids) + | set(graph.possible_tool_ids) + | set(graph.unbound_tool_ids) + ) + if catalog_ids != graph_catalog_ids: + raise SemanticConsistencyError("tool_catalog does not match binding graph partitions") + actions = { (action.tool_id, action.source_id, action.source_ref): action for action in report.action_surface_facts.actions @@ -90,6 +108,7 @@ def validate_semantic_consistency( if coverage.pass_eligible_actions != pass_eligible: raise SemanticConsistencyError("semantic pass_eligible_actions does not match assessments") identity_coverage = decision.evidence_coverage.identity_coverage + binding_coverage = decision.evidence_coverage.binding_coverage identity_eligible = sum( 1 for assessment in assessments.values() if assessment.identity.pass_eligible ) @@ -99,6 +118,8 @@ def validate_semantic_consistency( raise SemanticConsistencyError("identity pass_eligible_tools does not match assessments") if decision.decision == "passed" and ( coverage.gap_count + or binding_coverage.gap_count + or not graph.pass_eligible or coverage.review_concern_count or identity_coverage.gap_count or identity_eligible != len(assessments) diff --git a/src/agents_shipgate/inputs/crewai.py b/src/agents_shipgate/inputs/crewai.py index 97a8a0a7..b2aeca57 100644 --- a/src/agents_shipgate/inputs/crewai.py +++ b/src/agents_shipgate/inputs/crewai.py @@ -90,6 +90,11 @@ def __init__( self.discovered_tools: list[Tool] = [] self.list_vars: dict[str, list[str] | None] = {} self.agent_vars: set[str] = set() + self.call_targets = { + id(call): assignment_target(node) + for node in ordered_nodes(self.tree, (ast.Assign, ast.AnnAssign)) + if (call := assignment_call(node)) is not None + } self.warnings: list[str] = [] def extract(self) -> tuple[list[Tool], list[str]]: @@ -246,7 +251,13 @@ def _record_agent_or_crew(self, call: ast.Call) -> None: if call_kind == "Agent": tools_expr = keyword(call, "tools") names = self._resolve_tool_names(tools_expr) if tools_expr is not None else [] - record = {"source_ref": self.source_ref, "line": call.lineno, "tools": names or []} + record = { + "name": self.call_targets.get(id(call)) or "root", + "source_id": self.source_id, + "source_ref": self.source_ref, + "line": call.lineno, + "tools": names or [], + } self.artifacts.agents.append(record) if tools_expr is not None and names is None: self._dynamic( @@ -256,7 +267,13 @@ def _record_agent_or_crew(self, call: ast.Call) -> None: agents_expr = keyword(call, "agents") agents = _resolve_names(agents_expr) if agents_expr is not None else [] self.artifacts.crews.append( - {"source_ref": self.source_ref, "line": call.lineno, "agents": agents or []} + { + "name": self.call_targets.get(id(call)) or "crew", + "source_id": self.source_id, + "source_ref": self.source_ref, + "line": call.lineno, + "agents": agents or [], + } ) def _resolve_tool_names(self, node: ast.AST | None) -> list[str] | None: diff --git a/src/agents_shipgate/inputs/google_adk.py b/src/agents_shipgate/inputs/google_adk.py index d56fea7f..2f9c9c39 100644 --- a/src/agents_shipgate/inputs/google_adk.py +++ b/src/agents_shipgate/inputs/google_adk.py @@ -530,6 +530,7 @@ def extract(self) -> list[LoadedToolSource]: self.artifacts.agents.append( { "name": agent_name, + "source_id": self.source_id, "source_ref": self.source_ref, "instruction_present": bool(_kwarg_string(call, "instruction")), "instruction_preview": _kwarg_string(call, "instruction"), @@ -702,7 +703,11 @@ def _function_to_tool( output_schema={"type": _json_schema_type(return_type)} if return_type else {}, parameters=parameters, function_signature=signature, - annotations={"adk_agent_name": agent_name, "long_running": long_running}, + annotations={ + "adk_agent_name": agent_name, + "adk_agent_source_id": self.source_id, + "long_running": long_running, + }, auth=AuthInfo(source="google_adk_static"), extraction_confidence="medium", extraction={"method": "google_adk_python_ast", "confidence": "medium"}, @@ -747,6 +752,7 @@ def _extract_openapi_toolset(self, call: ast.Call, agent_name: str) -> list[Load for tool in loaded.tools: tool.annotations["adk_toolset"] = "OpenAPIToolset" tool.annotations["adk_agent_name"] = agent_name + tool.annotations["adk_agent_source_id"] = self.source_id return [loaded] def _extract_mcp_toolset(self, call: ast.Call, agent_name: str) -> list[LoadedToolSource]: @@ -782,6 +788,7 @@ def _extract_mcp_toolset(self, call: ast.Call, agent_name: str) -> list[LoadedTo for tool in loaded.tools: tool.annotations["adk_toolset"] = "McpToolset" tool.annotations["adk_agent_name"] = agent_name + tool.annotations["adk_agent_source_id"] = self.source_id return [loaded] def _record_agent_callbacks_plugins_subagents(self, call: ast.Call, agent_name: str) -> None: @@ -805,10 +812,21 @@ def _record_agent_callbacks_plugins_subagents(self, call: ast.Call, agent_name: ) elif keyword.arg == "sub_agents": sub_agent_count = len(keyword.value.elts) if isinstance(keyword.value, ast.List | ast.Tuple) else None + sub_agent_names = ( + [ + name + for item in keyword.value.elts + if (name := _qualified_name(item, self.aliases)) is not None + ] + if isinstance(keyword.value, ast.List | ast.Tuple) + else [] + ) self.artifacts.sub_agents.append( { "agent_name": agent_name, + "source_id": self.source_id, "sub_agent_count": sub_agent_count, + "sub_agents": sub_agent_names, "source_ref": f"{self.source_ref}:{call.lineno}", } ) diff --git a/src/agents_shipgate/inputs/langchain.py b/src/agents_shipgate/inputs/langchain.py index 189fd420..6af6c58d 100644 --- a/src/agents_shipgate/inputs/langchain.py +++ b/src/agents_shipgate/inputs/langchain.py @@ -92,6 +92,11 @@ def __init__( self.tool_vars: dict[str, Tool] = {} self.discovered_tools: list[Tool] = [] self.list_vars: dict[str, list[str] | None] = {} + self.call_targets = { + id(call): assignment_target(node) + for node in ordered_nodes(self.tree, (ast.Assign, ast.AnnAssign)) + if (call := assignment_call(node)) is not None + } self.warnings: list[str] = [] def extract(self) -> tuple[list[Tool], list[str]]: @@ -195,24 +200,37 @@ def _record_list_assignment(self, node: ast.Assign | ast.AnnAssign) -> None: def _record_tool_surface(self, call: ast.Call) -> None: call_kind = last_name(call.func) + agent_name = self.call_targets.get(id(call)) or "root" if call_kind in AGENT_BINDING_CALLS: tools_expr = keyword(call, "tools") if tools_expr is None and len(call.args) > 1: tools_expr = call.args[1] - self._record_binding("agent", call, tools_expr) + self._record_binding("agent", call, tools_expr, agent_name) elif call_kind == "ToolNode": - self._record_binding("tool_node", call, call.args[0] if call.args else None) + self._record_binding("tool_node", call, call.args[0] if call.args else None, agent_name) elif isinstance(call.func, ast.Attribute) and call.func.attr == "bind_tools": - self._record_binding("bind_tools", call, call.args[0] if call.args else keyword(call, "tools")) + self._record_binding("bind_tools", call, call.args[0] if call.args else keyword(call, "tools"), agent_name) - def _record_binding(self, kind: str, call: ast.Call, tools_expr: ast.AST | None) -> None: + def _record_binding( + self, + kind: str, + call: ast.Call, + tools_expr: ast.AST | None, + agent_name: str, + ) -> None: if tools_expr is None: return names = self._resolve_tool_names(tools_expr) if names is None: self._dynamic(kind, call.lineno, dynamic_reason(tools_expr), expr=tools_expr) return - record = {"source_ref": self.source_ref, "line": call.lineno, "tools": names} + record = { + "agent": agent_name, + "source_id": self.source_id, + "source_ref": self.source_ref, + "line": call.lineno, + "tools": names, + } if kind == "tool_node": self.artifacts.tool_nodes.append(record) else: diff --git a/src/agents_shipgate/inputs/n8n/_tools.py b/src/agents_shipgate/inputs/n8n/_tools.py index afc371ea..82b20e41 100644 --- a/src/agents_shipgate/inputs/n8n/_tools.py +++ b/src/agents_shipgate/inputs/n8n/_tools.py @@ -73,6 +73,7 @@ def _tools_from_tool_node( node_by_id: dict[str, _NodeItem], node_by_name: dict[str, _NodeItem], record_node_findings: bool = True, + bound_agent_names: list[str] | None = None, ) -> list[Tool]: # Late import: workflows.py owns the record builders + dynamic- # surface emitter. Loading them at call time keeps the @@ -105,7 +106,7 @@ def _tools_from_tool_node( ) if kind == "mcp_client_tool": - return _mcp_client_tools( + mcp_tools = _mcp_client_tools( item, source_id=source_id, source_path=source_path, @@ -115,6 +116,20 @@ def _tools_from_tool_node( artifacts=artifacts, warnings=warnings, ) + for tool in mcp_tools: + if not exposed_by_mcp: + for agent_name in bound_agent_names or []: + tool.annotations.setdefault("agent_bindings", []).append( + { + "agent": agent_name, + "source_id": workflow_id, + "edge_type": "workflow", + "source": source_path, + "source_pointer": f"{source_path}#node:{item.node_id}", + "complete": True, + } + ) + return mcp_tools if record_node_findings and kind == "workflow_tool": _record_workflow_resolution( item, @@ -135,6 +150,18 @@ def _tools_from_tool_node( source_type=source_type, exposed_by_mcp=exposed_by_mcp, ) + if not exposed_by_mcp: + for agent_name in bound_agent_names or []: + tool.annotations.setdefault("agent_bindings", []).append( + { + "agent": agent_name, + "source_id": workflow_id, + "edge_type": "workflow", + "source": source_path, + "source_pointer": f"{source_path}#node:{item.node_id}", + "complete": True, + } + ) if warning := tool_name_warning(tool.name): warnings.append(warning) _record_tool_artifact(kind, tool, item, source_path, workflow_id, artifacts) diff --git a/src/agents_shipgate/inputs/n8n/_workflows.py b/src/agents_shipgate/inputs/n8n/_workflows.py index c2d19377..85b6b20b 100644 --- a/src/agents_shipgate/inputs/n8n/_workflows.py +++ b/src/agents_shipgate/inputs/n8n/_workflows.py @@ -269,6 +269,11 @@ def _extract_workflow( for item in active_node_items if item.name and _node_kind(item.node_type) == "mcp_server_trigger" } + ai_agent_names = { + item.name + for item in active_node_items + if item.name and _node_kind(item.node_type) == "ai_agent" + } human_review_names = { item.name for item in active_node_items if item.name and _is_human_review_node(item) } @@ -346,6 +351,7 @@ def _extract_workflow( node_by_id=node_by_id, node_by_name=node_by_name, record_node_findings=index == 0, + bound_agent_names=sorted(set(targets) & ai_agent_names), ) tools.extend(extracted) diff --git a/src/agents_shipgate/inputs/openai_sdk_static.py b/src/agents_shipgate/inputs/openai_sdk_static.py index abbfb034..ad8aad1f 100644 --- a/src/agents_shipgate/inputs/openai_sdk_static.py +++ b/src/agents_shipgate/inputs/openai_sdk_static.py @@ -61,16 +61,18 @@ def load_openai_sdk_static_tools( toolkit_bounds.extend(file_bounds) elif path.suffix.lower() == ".py": tools, toolkit_bounds = _load_python_file(path, source, base_dir) + python_files = [path] else: raise InputParseError( f"OpenAI Agents SDK source must be a Python file or directory: {path}" ) + binding_warnings = _attach_agent_bindings(tools, python_files, source, base_dir) return LoadedToolSource( source_id=source.id, source_type="openai_agents_sdk", tools=tools, toolkit_bounds=toolkit_bounds, - warnings=_toolkit_binding_warnings(toolkit_bounds), + warnings=[*_toolkit_binding_warnings(toolkit_bounds), *binding_warnings], ) @@ -122,6 +124,155 @@ def _load_python_file( return tools, _detect_toolkit_bounds(tree, ref) +def _attach_agent_bindings( + tools: list[Tool], + paths: list[Path], + source: ToolSourceConfig, + base_dir: Path, +) -> list[str]: + """Attach exact, local-only ``Agent(..., tools=[...])`` wiring. + + This intentionally resolves only literal lists, names bound to literal + lists, and local/imported function-tool identifiers. Dynamic expressions + are preserved as partial evidence instead of being guessed. + """ + + warnings: list[str] = [] + tool_by_name = {tool.name: tool for tool in tools} + tool_by_name.update( + { + symbol: tool + for tool in tools + if isinstance((symbol := tool.annotations.get("python_symbol")), str) + } + ) + for path in paths: + tree = parse_python_file(path, label="OpenAI Agents SDK") + source_ref = display_path(path, base_dir) + list_vars: dict[str, list[str] | None] = {} + import_aliases: dict[str, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + for alias in node.names: + import_aliases[alias.asname or alias.name] = alias.name + if isinstance(node, (ast.Assign, ast.AnnAssign)): + target = _assignment_target(node) + value = node.value + if target and isinstance(value, (ast.List, ast.Tuple)): + list_vars[target] = _literal_names(value, import_aliases) + for node in ast.walk(tree): + if not isinstance(node, (ast.Assign, ast.AnnAssign)): + continue + target = _assignment_target(node) + call = node.value if isinstance(node.value, ast.Call) else None + if not target or call is None or _last_name(call.func) != "Agent": + continue + tools_expr = _keyword(call, "tools") + names = _resolve_name_list(tools_expr, list_vars, import_aliases) + pointer = f"{source_ref}:{call.lineno}" + if names is None: + reason = ( + f"OpenAI Agents SDK agent {target!r} at {pointer} uses a " + "dynamic tools expression; its binding graph is incomplete." + ) + warnings.append(reason) + for tool in tools: + if tool.source_id == source.id: + tool.annotations.setdefault("binding_surface_partial", []).append(reason) + continue + for name in names: + tool = tool_by_name.get(name) + if tool is None: + reason = ( + f"OpenAI Agents SDK agent {target!r} at {pointer} binds " + f"unresolved tool {name!r}." + ) + warnings.append(reason) + for candidate in tools: + if candidate.source_id == source.id: + candidate.annotations.setdefault("binding_surface_partial", []).append(reason) + continue + tool.annotations.setdefault("agent_bindings", []).append( + { + "agent": target, + "source_id": source.id, + "edge_type": "direct_tool", + "source": source_ref, + "source_pointer": pointer, + "complete": True, + } + ) + handoff_names = _resolve_name_list( + _keyword(call, "handoffs"), list_vars, import_aliases + ) + if handoff_names is None: + reason = f"OpenAI Agents SDK agent {target!r} has dynamic handoffs at {pointer}." + warnings.append(reason) + for tool in tools: + if tool.source_id == source.id: + tool.annotations.setdefault("binding_surface_partial", []).append(reason) + elif handoff_names: + anchor = next((tool_by_name.get(name) for name in names if tool_by_name.get(name)), None) + if anchor is not None: + anchor.annotations.setdefault("agent_handoffs", []).extend( + { + "source_agent": target, + "target_agent": handoff, + "source_id": source.id, + "edge_type": "handoff", + "source": source_ref, + "source_pointer": pointer, + "complete": True, + } + for handoff in handoff_names + ) + return list(dict.fromkeys(warnings)) + + +def _assignment_target(node: ast.Assign | ast.AnnAssign) -> str | None: + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + return targets[0].id if len(targets) == 1 and isinstance(targets[0], ast.Name) else None + + +def _last_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return None + + +def _keyword(call: ast.Call, name: str) -> ast.AST | None: + return next((item.value for item in call.keywords if item.arg == name), None) + + +def _literal_names( + value: ast.List | ast.Tuple, aliases: dict[str, str] +) -> list[str] | None: + names: list[str] = [] + for item in value.elts: + if not isinstance(item, ast.Name): + return None + names.append(aliases.get(item.id, item.id)) + return names + + +def _resolve_name_list( + value: ast.AST | None, + list_vars: dict[str, list[str] | None], + aliases: dict[str, str], +) -> list[str] | None: + if value is None: + return [] + if isinstance(value, (ast.List, ast.Tuple)): + return _literal_names(value, aliases) + if isinstance(value, ast.Name): + if value.id in list_vars: + return list_vars[value.id] + return [aliases.get(value.id, value.id)] + return None + + def _function_tool_decorator_names(tree: ast.Module) -> set[str]: names = set(DEFAULT_FUNCTION_TOOL_DECORATORS) for node in ast.walk(tree): @@ -173,6 +324,7 @@ def _function_to_tool( output_schema=function_output_schema(node), parameters=parameters, function_signature=function_signature(tool_name, parameters, node), + annotations={"python_symbol": node.name}, auth=AuthInfo(source="sdk_static"), extraction_confidence="medium", extraction={"method": "openai_agents_sdk_ast", "confidence": "medium"}, diff --git a/src/agents_shipgate/packet/json_packet.py b/src/agents_shipgate/packet/json_packet.py index 9ef3b4cd..53c86f44 100644 --- a/src/agents_shipgate/packet/json_packet.py +++ b/src/agents_shipgate/packet/json_packet.py @@ -83,12 +83,14 @@ def load_packet_json(payload: dict[str, Any] | str | bytes) -> EvidencePacket: version = payload_dict.get("packet_schema_version") legacy_version = ( - version if version in {"0.1", "0.2", "0.3", "0.4", "0.5", "0.6", "0.7", "0.8"} else None + version + if version in {"0.1", "0.2", "0.3", "0.4", "0.5", "0.6", "0.7", "0.8", "0.9"} + else None ) if version == "0.1": payload_dict = { **payload_dict, - "packet_schema_version": "0.9", + "packet_schema_version": "0.10", "tool_surface_diff": { "status": "not_declared", "enabled": False, @@ -103,37 +105,39 @@ def load_packet_json(payload: dict[str, Any] | str | bytes) -> EvidencePacket: _upgrade_evidence_matrix_v06(payload_dict) _upgrade_hitl_v07(payload_dict) elif version == "0.2": - payload_dict = {**payload_dict, "packet_schema_version": "0.9"} + payload_dict = {**payload_dict, "packet_schema_version": "0.10"} _upgrade_hitl_v03(payload_dict) _upgrade_action_surface_v05(payload_dict) _upgrade_evidence_matrix_v06(payload_dict) _upgrade_hitl_v07(payload_dict) elif version == "0.3": - payload_dict = {**payload_dict, "packet_schema_version": "0.9"} + payload_dict = {**payload_dict, "packet_schema_version": "0.10"} _upgrade_action_surface_v05(payload_dict) _upgrade_evidence_matrix_v06(payload_dict) _upgrade_hitl_v07(payload_dict) elif version == "0.4": - payload_dict = {**payload_dict, "packet_schema_version": "0.9"} + payload_dict = {**payload_dict, "packet_schema_version": "0.10"} _upgrade_action_surface_v05(payload_dict) _upgrade_evidence_matrix_v06(payload_dict) _upgrade_hitl_v07(payload_dict) elif version == "0.5": - payload_dict = {**payload_dict, "packet_schema_version": "0.9"} + payload_dict = {**payload_dict, "packet_schema_version": "0.10"} _upgrade_evidence_matrix_v06(payload_dict) _upgrade_hitl_v07(payload_dict) elif version == "0.6": - payload_dict = {**payload_dict, "packet_schema_version": "0.9"} + payload_dict = {**payload_dict, "packet_schema_version": "0.10"} _upgrade_hitl_v07(payload_dict) elif version == "0.7": - payload_dict = {**payload_dict, "packet_schema_version": "0.9"} + payload_dict = {**payload_dict, "packet_schema_version": "0.10"} elif version == "0.8": - payload_dict = {**payload_dict, "packet_schema_version": "0.9"} - elif version != "0.9": + payload_dict = {**payload_dict, "packet_schema_version": "0.10"} + elif version == "0.9": + payload_dict = {**payload_dict, "packet_schema_version": "0.10"} + elif version != "0.10": raise PacketSchemaError( "unsupported packet_schema_version: " f"{version!r}; expected '0.1', '0.2', '0.3', '0.4', '0.5', " - "'0.6', '0.7', '0.8', or '0.9'" + "'0.6', '0.7', '0.8', '0.9', or '0.10'" ) if legacy_version is not None: diff --git a/src/agents_shipgate/report/markdown.py b/src/agents_shipgate/report/markdown.py index 8cbf900f..c4f83672 100644 --- a/src/agents_shipgate/report/markdown.py +++ b/src/agents_shipgate/report/markdown.py @@ -133,6 +133,7 @@ def render_markdown_report( _append_frameworks(lines, report) _append_codex_plugin_surface(lines, report) _append_findings_by_category(lines, report.findings) + _append_binding_surface(lines, report) _append_inventory(lines, report) lines.extend(["", "## Disclaimer", "", DISCLAIMER, ""]) return "\n".join(lines) @@ -928,9 +929,9 @@ def _append_findings_by_category(lines: list[str], findings: list[Finding]) -> N def _append_inventory(lines: list[str], report: ReadinessReport) -> None: - lines.extend(["## Appendix: Normalized Tool Inventory", ""]) + lines.extend(["## Appendix: Root-Reachable Tool Inventory", ""]) if not report.tool_inventory: - lines.extend(["No tools were enumerated.", ""]) + lines.extend(["No tools were proven reachable from the root agent.", ""]) return lines.append("| Tool | Source | Risk Tags | Risk Confidence | Auth Scopes | Owner |") lines.append("| --- | --- | --- | --- | --- | --- |") @@ -947,6 +948,41 @@ def _append_inventory(lines: list[str], report: ReadinessReport) -> None: lines.append("") +def _append_binding_surface(lines: list[str], report: ReadinessReport) -> None: + graph = report.binding_surface_facts + lines.extend(["## Agent Binding Surface", ""]) + lines.append(f"Status: {_safe_markdown_text(graph.status)}") + lines.append(f"Root agent: {_safe_markdown_text(graph.root_agent_id or 'unresolved')}") + lines.append(f"Pass eligible: {str(graph.pass_eligible).lower()}") + lines.append( + "Catalog partition: " + f"{len(graph.reachable_tool_ids)} reachable, " + f"{len(graph.possible_tool_ids)} possible, " + f"{len(graph.unbound_tool_ids)} unbound" + ) + if graph.issues: + lines.append("") + lines.append("Binding gaps:") + for issue in graph.issues: + lines.append( + f"- `{_safe_markdown_text(issue.kind)}` — " + f"{_safe_markdown_text(issue.message)}" + ) + lines.append("") + + if report.tool_catalog and len(report.tool_catalog) != len(report.tool_inventory): + lines.extend(["### Unbound Catalog Entries", ""]) + reachable = set(graph.reachable_tool_ids) + for tool in report.tool_catalog: + if tool.get("tool_id") in reachable: + continue + lines.append( + f"- `{_safe_markdown_text(tool.get('name') or '-')}` " + f"({_safe_markdown_text(tool.get('source_type') or '-')})" + ) + lines.append("") + + def _human_status(status: str) -> str: return status.replace("_", " ").capitalize() diff --git a/src/agents_shipgate/report/summary_text.py b/src/agents_shipgate/report/summary_text.py index af81d930..9aa8ff54 100644 --- a/src/agents_shipgate/report/summary_text.py +++ b/src/agents_shipgate/report/summary_text.py @@ -10,6 +10,13 @@ def evidence_coverage_text(evidence: EvidenceCoverageDecision) -> str: if evidence.source_warning_count: extras.append(f"{evidence.source_warning_count} source warning(s)") semantic = evidence.semantic_coverage + binding = evidence.binding_coverage + if binding.gap_count: + extras.append(f"{binding.gap_count} binding evidence gap(s)") + if binding.total_catalog_tools: + extras.append( + f"{binding.reachable_tools}/{binding.total_catalog_tools} catalog tools reachable" + ) if semantic.gap_count: extras.append(f"{semantic.gap_count} semantic evidence gap(s)") if semantic.review_concern_count: diff --git a/src/agents_shipgate/schemas/bindings.py b/src/agents_shipgate/schemas/bindings.py new file mode 100644 index 00000000..662a6b9b --- /dev/null +++ b/src/agents_shipgate/schemas/bindings.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from agents_shipgate.schemas.common import Confidence, ProvenanceKind + +BindingStatus = Literal["declared", "structural", "partial", "unknown", "conflicting"] +BindingEdgeType = Literal[ + "direct_tool", + "tool_node", + "toolset", + "workflow", + "subagent", + "handoff", +] + + +class AgentBindingNode(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + agent_id: str + name: str + source_id: str | None = None + source_ref: str | None = None + source_pointer: str | None = None + + +class AgentToolBindingEdge(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + agent_id: str + tool_id: str + edge_type: BindingEdgeType + confidence: Confidence + provenance_kind: ProvenanceKind + source: str + source_pointer: str | None = None + complete: bool = True + + +class AgentHandoffBindingEdge(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + source_agent_id: str + target_agent_id: str + edge_type: Literal["subagent", "handoff"] + confidence: Confidence + provenance_kind: ProvenanceKind + source: str + source_pointer: str | None = None + complete: bool = True + + +class AgentBindingIssue(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: Literal[ + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation", + ] + message: str + agent_id: str | None = None + tool_id: str | None = None + source: str | None = None + source_pointer: str | None = None + + +class AgentBindingGraphAssessment(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + root_agent_id: str | None = None + status: BindingStatus + agents: list[AgentBindingNode] = Field(default_factory=list) + tool_edges: list[AgentToolBindingEdge] = Field(default_factory=list) + handoff_edges: list[AgentHandoffBindingEdge] = Field(default_factory=list) + reachable_tool_ids: list[str] = Field(default_factory=list) + possible_tool_ids: list[str] = Field(default_factory=list) + unbound_tool_ids: list[str] = Field(default_factory=list) + issues: list[AgentBindingIssue] = Field(default_factory=list) + pass_eligible: bool = False + + +class BindingSurfaceDiff(BaseModel): + model_config = ConfigDict(extra="forbid") + + enabled: bool = False + base_report_schema_version: str | None = None + added_reachable_tool_ids: list[str] = Field(default_factory=list) + removed_reachable_tool_ids: list[str] = Field(default_factory=list) + added_handoffs: list[str] = Field(default_factory=list) + removed_handoffs: list[str] = Field(default_factory=list) + notes: list[str] = Field(default_factory=list) + + +__all__ = [ + "AgentBindingGraphAssessment", + "AgentBindingIssue", + "AgentBindingNode", + "AgentHandoffBindingEdge", + "AgentToolBindingEdge", + "BindingEdgeType", + "BindingStatus", + "BindingSurfaceDiff", +] diff --git a/src/agents_shipgate/schemas/capabilities.py b/src/agents_shipgate/schemas/capabilities.py index ad8ded98..5e514bd6 100644 --- a/src/agents_shipgate/schemas/capabilities.py +++ b/src/agents_shipgate/schemas/capabilities.py @@ -14,9 +14,9 @@ from agents_shipgate.schemas.semantic import ToolSemanticEvidence from agents_shipgate.schemas.surfaces import ActionEffect -CAPABILITY_LOCK_SCHEMA_VERSION = "0.4" -CAPABILITY_LOCK_DIFF_SCHEMA_VERSION = "0.5" -CAPABILITY_STANDARD_VERSION = "0.3" +CAPABILITY_LOCK_SCHEMA_VERSION = "0.5" +CAPABILITY_LOCK_DIFF_SCHEMA_VERSION = "0.6" +CAPABILITY_STANDARD_VERSION = "0.4" CapabilityEvidenceProvenanceKind = Literal[ "static_declaration", "ast_extraction", @@ -109,6 +109,7 @@ class CapabilityHashes(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") identity_hash: str + binding_hash: str = "legacy_binding_unknown" effect_hash: str authority_hash: str control_hash: str @@ -210,7 +211,7 @@ class CapabilityLockHashes(BaseModel): class CapabilityLockFileV1(BaseModel): model_config = ConfigDict(extra="forbid") - capability_lock_schema_version: Literal["0.4"] = CAPABILITY_LOCK_SCHEMA_VERSION + capability_lock_schema_version: Literal["0.5"] = CAPABILITY_LOCK_SCHEMA_VERSION experimental: Literal[False] = False cli_version: str source: CapabilityLockSource @@ -262,7 +263,7 @@ class CapabilityLockChangedFact(BaseModel): class CapabilityLockDiffV1(BaseModel): model_config = ConfigDict(extra="forbid") - capability_lock_diff_schema_version: Literal["0.5"] = ( + capability_lock_diff_schema_version: Literal["0.6"] = ( CAPABILITY_LOCK_DIFF_SCHEMA_VERSION ) experimental: Literal[False] = False diff --git a/src/agents_shipgate/schemas/capability_semantics.py b/src/agents_shipgate/schemas/capability_semantics.py index fa376df0..20ebffe8 100644 --- a/src/agents_shipgate/schemas/capability_semantics.py +++ b/src/agents_shipgate/schemas/capability_semantics.py @@ -7,6 +7,7 @@ CapabilityHashName = Literal[ "identity_hash", + "binding_hash", "effect_hash", "authority_hash", "control_hash", diff --git a/src/agents_shipgate/schemas/contract.py b/src/agents_shipgate/schemas/contract.py index d5a5d63c..c913e966 100644 --- a/src/agents_shipgate/schemas/contract.py +++ b/src/agents_shipgate/schemas/contract.py @@ -33,7 +33,7 @@ from agents_shipgate.schemas.verifier import VerifierArtifact from agents_shipgate.schemas.verify_run import VERIFY_RUN_SCHEMA_VERSION -CONTRACT_VERSION: Literal["12"] = "12" +CONTRACT_VERSION: Literal["13"] = "13" GATING_SIGNAL: Literal["release_decision.decision"] = "release_decision.decision" AGENT_RESULT_SCHEMA_VERSION: Literal["agent_result_v1"] = "agent_result_v1" AGENT_RESULT_SCHEMA_PATH: Literal["docs/agent-result-schema.v1.json"] = ( @@ -251,6 +251,7 @@ # never invent or auto-fill either declaration. "action_effect", "action_authority", + "agent_binding", "approval", "confirmation", "idempotency", diff --git a/src/agents_shipgate/schemas/manifest/__init__.py b/src/agents_shipgate/schemas/manifest/__init__.py index b73bdf9a..8c3b0ca8 100644 --- a/src/agents_shipgate/schemas/manifest/__init__.py +++ b/src/agents_shipgate/schemas/manifest/__init__.py @@ -41,6 +41,12 @@ AgentConfig, AgentSdkConfig, ) +from agents_shipgate.schemas.manifest.agent_bindings import ( + AgentBindingDeclarationConfig, + AgentBindingRootConfig, + AgentBindingsConfig, + BoundToolSelectorConfig, +) from agents_shipgate.schemas.manifest.anthropic import AnthropicConfig from agents_shipgate.schemas.manifest.baseline import ( BaselineConfig, @@ -126,6 +132,10 @@ # agent "AgentConfig", "AgentSdkConfig", + "AgentBindingDeclarationConfig", + "AgentBindingRootConfig", + "AgentBindingsConfig", + "BoundToolSelectorConfig", # root "AgentsShipgateManifest", # anthropic diff --git a/src/agents_shipgate/schemas/manifest/agent_bindings.py b/src/agents_shipgate/schemas/manifest/agent_bindings.py new file mode 100644 index 00000000..556a95d8 --- /dev/null +++ b/src/agents_shipgate/schemas/manifest/agent_bindings.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field, field_validator, model_validator + +from agents_shipgate.schemas.manifest._common import STRICT_MODEL_CONFIG + + +class AgentBindingRootConfig(BaseModel): + """Exact selector for the statically reviewed root agent.""" + + model_config = STRICT_MODEL_CONFIG + + object: str + source_id: str | None = None + + @field_validator("object", "source_id") + @classmethod + def require_non_blank(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + if not normalized: + raise ValueError("agent binding root selectors require non-blank values") + return normalized + + +class BoundToolSelectorConfig(BaseModel): + """One canonical tool selector inside a reviewed binding declaration.""" + + model_config = STRICT_MODEL_CONFIG + + tool: str + tool_id: str | None = None + provider: str | None = None + source_type: str | None = None + source_id: str | None = None + + @field_validator("tool", "tool_id", "provider", "source_type", "source_id") + @classmethod + def require_non_blank(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + if not normalized: + raise ValueError("agent binding tool selectors require non-blank values") + return normalized + + +class AgentBindingDeclarationConfig(BaseModel): + """Reviewed, closed-world tool and handoff set for one agent.""" + + model_config = STRICT_MODEL_CONFIG + + agent: str + complete: Literal[True] = True + tools: list[BoundToolSelectorConfig] = Field(default_factory=list) + handoffs: list[str] = Field(default_factory=list) + reason: str + + @field_validator("agent", "reason") + @classmethod + def require_non_blank(cls, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("agent binding declarations require non-blank agent and reason") + return normalized + + @field_validator("handoffs") + @classmethod + def validate_handoffs(cls, values: list[str]) -> list[str]: + normalized = [value.strip() for value in values] + if any(not value for value in normalized): + raise ValueError("agent binding handoffs require non-blank values") + if len(set(normalized)) != len(normalized): + raise ValueError("agent binding handoffs contain duplicates") + return normalized + + @model_validator(mode="after") + def validate_tools(self) -> AgentBindingDeclarationConfig: + keys = [ + ( + selector.tool_id or "", + selector.tool, + selector.provider or "", + selector.source_type or "", + selector.source_id or "", + ) + for selector in self.tools + ] + if len(set(keys)) != len(keys): + raise ValueError("agent binding declarations contain duplicate tool selectors") + return self + + +class AgentBindingsConfig(BaseModel): + model_config = STRICT_MODEL_CONFIG + + root: AgentBindingRootConfig | None = None + declarations: list[AgentBindingDeclarationConfig] = Field(default_factory=list) + + @model_validator(mode="after") + def require_unique_agent_declarations(self) -> AgentBindingsConfig: + agents = [declaration.agent for declaration in self.declarations] + if len(set(agents)) != len(agents): + raise ValueError("agent_bindings.declarations[].agent values must be unique") + return self + + +__all__ = [ + "AgentBindingDeclarationConfig", + "AgentBindingRootConfig", + "AgentBindingsConfig", + "BoundToolSelectorConfig", +] diff --git a/src/agents_shipgate/schemas/manifest/root.py b/src/agents_shipgate/schemas/manifest/root.py index e25040e5..ceb71813 100644 --- a/src/agents_shipgate/schemas/manifest/root.py +++ b/src/agents_shipgate/schemas/manifest/root.py @@ -6,6 +6,7 @@ from agents_shipgate.schemas.manifest._common import STRICT_MODEL_CONFIG from agents_shipgate.schemas.manifest.action_surface import ActionSurfaceConfig from agents_shipgate.schemas.manifest.agent import AgentConfig +from agents_shipgate.schemas.manifest.agent_bindings import AgentBindingsConfig from agents_shipgate.schemas.manifest.anthropic import AnthropicConfig from agents_shipgate.schemas.manifest.baseline import BaselineConfig from agents_shipgate.schemas.manifest.checks import ChecksConfig @@ -55,6 +56,7 @@ class AgentsShipgateManifest(BaseModel): risk_overrides: RiskOverridesConfig = Field(default_factory=RiskOverridesConfig) checks: ChecksConfig = Field(default_factory=ChecksConfig) action_surface: ActionSurfaceConfig = Field(default_factory=ActionSurfaceConfig) + agent_bindings: AgentBindingsConfig = Field(default_factory=AgentBindingsConfig) ci: CiConfig = Field(default_factory=CiConfig) baseline: BaselineConfig = Field(default_factory=BaselineConfig) output: OutputConfig = Field(default_factory=OutputConfig) diff --git a/src/agents_shipgate/schemas/packet.py b/src/agents_shipgate/schemas/packet.py index 0ea109b2..7631a98b 100644 --- a/src/agents_shipgate/schemas/packet.py +++ b/src/agents_shipgate/schemas/packet.py @@ -359,7 +359,7 @@ class EvidencePacket(BaseModel): # human_in_the_loop. release_decision.decision remains the only gate. # v0.8: release-decision evidence coverage carries the additive # evidence-backed semantic coverage and gap remediation contract. - packet_schema_version: Literal["0.9"] = "0.9" + packet_schema_version: Literal["0.10"] = "0.10" generated_at: str | None = None run_id: str project: dict[str, Any] = Field(default_factory=dict) diff --git a/src/agents_shipgate/schemas/report.py b/src/agents_shipgate/schemas/report.py index dce90c2d..0c038b95 100644 --- a/src/agents_shipgate/schemas/report.py +++ b/src/agents_shipgate/schemas/report.py @@ -4,6 +4,10 @@ from pydantic import BaseModel, ConfigDict, Field +from agents_shipgate.schemas.bindings import ( + AgentBindingGraphAssessment, + BindingSurfaceDiff, +) from agents_shipgate.schemas.capability_change import ( CapabilityChangeBlock, EffectivePolicy, @@ -322,6 +326,12 @@ class EvidenceGapAction(BaseModel): "provide_tool_binding", "resolve_tool_identity_conflict", "regenerate_identity_artifact", + "declare_agent_root", + "declare_agent_bindings", + "provide_static_binding_source", + "provide_complete_binding_graph", + "resolve_binding_conflict", + "regenerate_binding_artifact", ] command: str | None = None path: str | None = None @@ -367,6 +377,14 @@ class EvidenceGap(BaseModel): "ambiguous_tool_selector", "ambiguous_legacy_tool_identity", "invalid_tool_binding", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation", ] subject: str source_type: str | None = None @@ -403,6 +421,16 @@ class IdentityCoverageDecision(BaseModel): reason_counts: dict[str, int] = Field(default_factory=dict) +class BindingCoverageDecision(BaseModel): + total_catalog_tools: int = 0 + reachable_tools: int = 0 + possible_tools: int = 0 + unbound_tools: int = 0 + pass_eligible: bool = False + gap_count: int = 0 + reason_counts: dict[str, int] = Field(default_factory=dict) + + class EvidenceCoverageDecision(BaseModel): level: str human_review_recommended: bool @@ -417,6 +445,7 @@ class EvidenceCoverageDecision(BaseModel): # always populate it from Tool.semantic_assessment. semantic_coverage: SemanticCoverageDecision = Field(default_factory=SemanticCoverageDecision) identity_coverage: IdentityCoverageDecision = Field(default_factory=IdentityCoverageDecision) + binding_coverage: BindingCoverageDecision = Field(default_factory=BindingCoverageDecision) class BaselineDelta(BaseModel): @@ -911,7 +940,7 @@ class ReadinessReport(BaseModel): # The release gate is unchanged; these are org-governance audit fields. # v0.29: additive semantic assessments and zero-tolerance semantic # evidence coverage make ``passed`` evidence-backed. - report_schema_version: str = "0.30" + report_schema_version: str = "0.31" run_id: str # v0.6 (per C13): absolute path to the directory containing # shipgate.yaml. apply-patches uses this to enforce a containment @@ -943,6 +972,14 @@ class ReadinessReport(BaseModel): # static tool surface plus optional manifest action declarations. action_surface_facts: ActionSurfaceFacts = Field(default_factory=ActionSurfaceFacts) action_surface_diff: ActionSurfaceDiff = Field(default_factory=ActionSurfaceDiff) + binding_surface_facts: AgentBindingGraphAssessment = Field( + default_factory=lambda: AgentBindingGraphAssessment( + root_agent_id="legacy_direct", + status="structural", + pass_eligible=True, + ) + ) + binding_surface_diff: BindingSurfaceDiff = Field(default_factory=BindingSurfaceDiff) capability_runtime_evidence: CapabilityRuntimeEvidence = Field( default_factory=CapabilityRuntimeEvidence ) @@ -964,6 +1001,7 @@ class ReadinessReport(BaseModel): # constructing minimal reports keep working. loaded_adapters: list[dict[str, Any]] = Field(default_factory=list) tool_inventory: list[dict[str, Any]] = Field(default_factory=list) + tool_catalog: list[dict[str, Any]] = Field(default_factory=list) source_warnings: list[str] = Field(default_factory=list) # v0.12: top-level agent summary. Deterministic projection of # release_decision + findings[].agent_action. Optional at Python diff --git a/src/agents_shipgate/schemas/safety_qualification.py b/src/agents_shipgate/schemas/safety_qualification.py index 182025a8..967abe3e 100644 --- a/src/agents_shipgate/schemas/safety_qualification.py +++ b/src/agents_shipgate/schemas/safety_qualification.py @@ -9,15 +9,21 @@ from agents_shipgate.schemas.common import ReleaseDecisionStatus from agents_shipgate.schemas.disclaimers import STATIC_VERDICT_DISCLAIMER -SAFETY_CORPUS_SCHEMA_VERSION = "shipgate.safety_corpus/v1" -SAFETY_RECEIPT_INDEX_SCHEMA_VERSION = "shipgate.safety_receipt_index/v1" -SAFETY_QUALIFICATION_SCHEMA_VERSION = "shipgate.safety_qualification/v1" +SAFETY_CORPUS_SCHEMA_VERSION = "shipgate.safety_corpus/v2" +SAFETY_RECEIPT_INDEX_SCHEMA_VERSION = "shipgate.safety_receipt_index/v2" +SAFETY_QUALIFICATION_SCHEMA_VERSION = "shipgate.safety_qualification/v2" SafetyProfile = Literal[ "mcp", "openapi", "explicit_framework_inventory", "coding_agent_trust_roots", + "mcp_openapi_declared_binding", + "openai_agents_sdk", + "langchain_crewai", + "google_adk", + "n8n", + "multi_agent_handoffs", ] SafetyCaseOrigin = Literal[ "real_history", @@ -184,13 +190,18 @@ class FrozenSafetyCorpusV1(BaseModel): model_config = ConfigDict(extra="forbid") - schema_version: Literal["shipgate.safety_corpus/v1"] = SAFETY_CORPUS_SCHEMA_VERSION + schema_version: Literal["shipgate.safety_corpus/v2"] = SAFETY_CORPUS_SCHEMA_VERSION corpus_id: str = Field(min_length=1) labels_frozen_before_evaluation: Literal[True] outputs_hidden_from_labelers: Literal[True] frozen_labels_sha256: str = Field(pattern=SHA256_PATTERN) cases: list[SafetyCorpusCaseV1] = Field(min_length=1) + @field_validator("schema_version", mode="before") + @classmethod + def _upgrade_v1_schema(cls, value: str) -> str: + return SAFETY_CORPUS_SCHEMA_VERSION if value == "shipgate.safety_corpus/v1" else value + @field_validator("corpus_id") @classmethod def _corpus_id_is_not_blank(cls, value: str) -> str: @@ -254,7 +265,7 @@ class SafetyReceiptIndexV1(BaseModel): model_config = ConfigDict(extra="forbid") - schema_version: Literal["shipgate.safety_receipt_index/v1"] = ( + schema_version: Literal["shipgate.safety_receipt_index/v2"] = ( SAFETY_RECEIPT_INDEX_SCHEMA_VERSION ) wheel_sha256: str = Field(pattern=SHA256_PATTERN) @@ -265,6 +276,15 @@ class SafetyReceiptIndexV1(BaseModel): labels_frozen_before_receipts: Literal[True] receipts: list[SafetyReceiptEntryV1] = Field(min_length=1) + @field_validator("schema_version", mode="before") + @classmethod + def _upgrade_v1_schema(cls, value: str) -> str: + return ( + SAFETY_RECEIPT_INDEX_SCHEMA_VERSION + if value == "shipgate.safety_receipt_index/v1" + else value + ) + @field_validator("engine_version") @classmethod def _engine_version_is_not_blank(cls, value: str) -> str: @@ -316,28 +336,46 @@ def _report_schema_version_is_not_blank(cls, value: str) -> str: def production_safety_requirements() -> SafetyQualificationRequirementsV1: profile_counts = { - "mcp": { - "passed": 10, - "review_required": 7, - "insufficient_evidence": 8, - "blocked": 10, + "mcp_openapi_declared_binding": { + "passed": 6, + "review_required": 4, + "insufficient_evidence": 4, + "blocked": 6, }, - "openapi": { - "passed": 10, - "review_required": 7, - "insufficient_evidence": 8, - "blocked": 10, + "openai_agents_sdk": { + "passed": 5, + "review_required": 3, + "insufficient_evidence": 3, + "blocked": 4, }, - "explicit_framework_inventory": { + "langchain_crewai": { "passed": 5, "review_required": 3, + "insufficient_evidence": 3, + "blocked": 4, + }, + "google_adk": { + "passed": 3, + "review_required": 2, "insufficient_evidence": 2, + "blocked": 3, + }, + "n8n": { + "passed": 3, + "review_required": 2, + "insufficient_evidence": 2, + "blocked": 3, + }, + "multi_agent_handoffs": { + "passed": 4, + "review_required": 3, + "insufficient_evidence": 3, "blocked": 5, }, "coding_agent_trust_roots": { - "passed": 5, + "passed": 4, "review_required": 3, - "insufficient_evidence": 2, + "insufficient_evidence": 3, "blocked": 5, }, } @@ -360,7 +398,7 @@ def production_safety_requirements() -> SafetyQualificationRequirementsV1: minimum_blocked_exact=30, minimum_review_exact=19, minimum_insufficient_evidence_exact=19, - required_report_schema_version="0.30", + required_report_schema_version="0.31", ) @@ -472,7 +510,7 @@ class SafetyQualificationResultV1(BaseModel): model_config = ConfigDict(extra="forbid") - schema_version: Literal["shipgate.safety_qualification/v1"] = ( + schema_version: Literal["shipgate.safety_qualification/v2"] = ( SAFETY_QUALIFICATION_SCHEMA_VERSION ) qualification_tier: QualificationTier @@ -490,6 +528,15 @@ class SafetyQualificationResultV1(BaseModel): cases: list[SafetyQualificationCaseResultV1] failures: list[SafetyQualificationFailureV1] + @field_validator("schema_version", mode="before") + @classmethod + def _upgrade_v1_schema(cls, value: str) -> str: + return ( + SAFETY_QUALIFICATION_SCHEMA_VERSION + if value == "shipgate.safety_qualification/v1" + else value + ) + @model_validator(mode="after") def _sort_output(self) -> SafetyQualificationResultV1: self.strata.sort(key=lambda item: (item.profile, item.expected_decision)) diff --git a/src/agents_shipgate/schemas/semantic.py b/src/agents_shipgate/schemas/semantic.py index b917ca08..5549fcbd 100644 --- a/src/agents_shipgate/schemas/semantic.py +++ b/src/agents_shipgate/schemas/semantic.py @@ -26,7 +26,7 @@ "code_execution", "identity_access", ] -SemanticDimension = Literal["identity", "effect", "authority"] +SemanticDimension = Literal["identity", "binding", "effect", "authority"] IdentityEvidenceStatus = Literal[ "declared", "structural", @@ -65,6 +65,14 @@ "partial_authority_evidence", "conflicting_authority_evidence", "invalid_semantic_annotation", + "missing_binding_evidence", + "partial_binding_evidence", + "conflicting_binding_evidence", + "ambiguous_root_agent", + "unresolved_agent_binding", + "unresolved_bound_tool", + "incomplete_handoff_graph", + "invalid_binding_annotation", ] @@ -125,6 +133,28 @@ class ToolIdentityEvidence(BaseModel): pass_eligible: bool +class BindingSemanticEvidence(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + status: Literal["declared", "structural", "partial", "unknown", "conflicting"] + confidence: Confidence + root_agent_id: str | None = None + reachable_path: list[str] = Field(default_factory=list) + claims: list[SemanticClaimEvidence] = Field(default_factory=list) + issues: list[SemanticIssueEvidence] = Field(default_factory=list) + pass_eligible: bool = False + + +def _legacy_direct_binding() -> BindingSemanticEvidence: + return BindingSemanticEvidence( + status="structural", + confidence="high", + root_agent_id="legacy_direct", + reachable_path=["legacy_direct"], + pass_eligible=True, + ) + + class ToolSemanticEvidence(BaseModel): """Public semantic assessment attached to action and capability facts.""" @@ -132,6 +162,7 @@ class ToolSemanticEvidence(BaseModel): conservative_effect: SemanticActionEffect identity: ToolIdentityEvidence | None = None + binding: BindingSemanticEvidence = Field(default_factory=_legacy_direct_binding) effect: EffectSemanticEvidence authority: AuthoritySemanticEvidence pass_eligible: bool @@ -141,6 +172,7 @@ class ToolSemanticEvidence(BaseModel): "AuthorityEvidenceStatus", "AuthorityMode", "AuthoritySemanticEvidence", + "BindingSemanticEvidence", "EffectEvidenceStatus", "EffectSemanticEvidence", "IdentityEvidenceStatus", diff --git a/src/agents_shipgate/schemas/surfaces.py b/src/agents_shipgate/schemas/surfaces.py index e3520832..7b80e780 100644 --- a/src/agents_shipgate/schemas/surfaces.py +++ b/src/agents_shipgate/schemas/surfaces.py @@ -349,7 +349,7 @@ class ActionFact(BaseModel): class ActionSurfaceFacts(BaseModel): model_config = ConfigDict(extra="forbid") - snapshot_version: str = "0.2" + snapshot_version: str = "0.3" actions: list[ActionFact] = Field(default_factory=list) diff --git a/tests/golden/agent_protocol/claude-code-block-stop.json b/tests/golden/agent_protocol/claude-code-block-stop.json index 704820ef..aedcaaaa 100644 --- a/tests/golden/agent_protocol/claude-code-block-stop.json +++ b/tests/golden/agent_protocol/claude-code-block-stop.json @@ -3,7 +3,7 @@ "agent": "claude-code", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "claude-code" diff --git a/tests/golden/agent_protocol/codex-block-stop.json b/tests/golden/agent_protocol/codex-block-stop.json index ca1fd00e..e4dbf573 100644 --- a/tests/golden/agent_protocol/codex-block-stop.json +++ b/tests/golden/agent_protocol/codex-block-stop.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/agent_protocol/codex-repair-after.json b/tests/golden/agent_protocol/codex-repair-after.json index 1302b534..ea73b334 100644 --- a/tests/golden/agent_protocol/codex-repair-after.json +++ b/tests/golden/agent_protocol/codex-repair-after.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/agent_protocol/codex-repair-before.json b/tests/golden/agent_protocol/codex-repair-before.json index b9395fa3..85e4ef06 100644 --- a/tests/golden/agent_protocol/codex-repair-before.json +++ b/tests/golden/agent_protocol/codex-repair-before.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/agent_protocol/cursor-block-stop.json b/tests/golden/agent_protocol/cursor-block-stop.json index f401dde6..b74364b1 100644 --- a/tests/golden/agent_protocol/cursor-block-stop.json +++ b/tests/golden/agent_protocol/cursor-block-stop.json @@ -3,7 +3,7 @@ "agent": "cursor", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "cursor" diff --git a/tests/golden/codex_boundary_result/agents_requirement_removed.json b/tests/golden/codex_boundary_result/agents_requirement_removed.json index b23b32e3..245f0a25 100644 --- a/tests/golden/codex_boundary_result/agents_requirement_removed.json +++ b/tests/golden/codex_boundary_result/agents_requirement_removed.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/codex_boundary_result/docs_only.json b/tests/golden/codex_boundary_result/docs_only.json index 5fb526e0..88a031a7 100644 --- a/tests/golden/codex_boundary_result/docs_only.json +++ b/tests/golden/codex_boundary_result/docs_only.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/codex_boundary_result/github_action_removed.json b/tests/golden/codex_boundary_result/github_action_removed.json index 6b215771..52343697 100644 --- a/tests/golden/codex_boundary_result/github_action_removed.json +++ b/tests/golden/codex_boundary_result/github_action_removed.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/codex_boundary_result/malformed_toml.json b/tests/golden/codex_boundary_result/malformed_toml.json index daa523b5..f4032852 100644 --- a/tests/golden/codex_boundary_result/malformed_toml.json +++ b/tests/golden/codex_boundary_result/malformed_toml.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/codex_boundary_result/mcp_auto_approve_write.json b/tests/golden/codex_boundary_result/mcp_auto_approve_write.json index 1ad86425..0fad53d0 100644 --- a/tests/golden/codex_boundary_result/mcp_auto_approve_write.json +++ b/tests/golden/codex_boundary_result/mcp_auto_approve_write.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/codex_boundary_result/network_wildcard.json b/tests/golden/codex_boundary_result/network_wildcard.json index df25304d..ed87b26c 100644 --- a/tests/golden/codex_boundary_result/network_wildcard.json +++ b/tests/golden/codex_boundary_result/network_wildcard.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/codex_boundary_result/python_refactor.json b/tests/golden/codex_boundary_result/python_refactor.json index 74316847..6424a6f3 100644 --- a/tests/golden/codex_boundary_result/python_refactor.json +++ b/tests/golden/codex_boundary_result/python_refactor.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/golden/codex_boundary_result/unknown_permission_key.json b/tests/golden/codex_boundary_result/unknown_permission_key.json index dceb1c6b..1478e8ed 100644 --- a/tests/golden/codex_boundary_result/unknown_permission_key.json +++ b/tests/golden/codex_boundary_result/unknown_permission_key.json @@ -3,7 +3,7 @@ "agent": "codex", "tool": { "name": "agents-shipgate", - "version": "0.16.0b1" + "version": "0.16.0b2" }, "subject": { "agent": "codex" diff --git a/tests/test_action_surface_diff.py b/tests/test_action_surface_diff.py index a07aa838..b875d47b 100644 --- a/tests/test_action_surface_diff.py +++ b/tests/test_action_surface_diff.py @@ -1152,6 +1152,13 @@ def test_scan_diff_from_prior_report_does_not_launder_financial_keyword_into_blo - id: billing-api type: openapi path: openapi.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: [{tool: refund_customer, source_id: billing-api}] + handoffs: [] + reason: reviewed action-diff fixture binding """ (base / "shipgate.yaml").write_text(manifest_text, encoding="utf-8") (head / "shipgate.yaml").write_text(manifest_text, encoding="utf-8") @@ -1225,6 +1232,13 @@ def test_action_surface_diff_can_use_v04_baseline(tmp_path): - id: billing-api type: openapi path: openapi.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: [{tool: refund_customer, source_id: billing-api}] + handoffs: [] + reason: reviewed baseline-diff fixture binding """ (base / "shipgate.yaml").write_text(manifest_text, encoding="utf-8") (head / "shipgate.yaml").write_text(manifest_text, encoding="utf-8") @@ -1309,6 +1323,13 @@ def test_action_policy_blocks_strict_ci_and_supports_suppression(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: [{tool: lookup, source_id: tools}] + handoffs: [] + reason: reviewed action-policy fixture binding action_surface: actions: - tool: lookup @@ -1386,6 +1407,13 @@ def test_user_policy_evaluates_full_surface_when_diff_enabled(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: [{tool: lookup, source_id: tools}] + handoffs: [] + reason: reviewed action-policy diff binding """ head_manifest = ( base_manifest diff --git a/tests/test_agent_bindings.py b/tests/test_agent_bindings.py new file mode 100644 index 00000000..37933e6d --- /dev/null +++ b/tests/test_agent_bindings.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +from agents_shipgate.core.agent_bindings import resolve_agent_binding_graph +from agents_shipgate.core.artifacts import ArtifactBag +from agents_shipgate.core.domain import AuthInfo, Tool +from agents_shipgate.schemas.manifest import AgentsShipgateManifest + + +def _tool(name: str, *, source_id: str = "catalog", agent: str | None = None) -> Tool: + annotations = {"readOnlyHint": True} + if agent is not None: + annotations["agent_bindings"] = [ + { + "agent": agent, + "source_id": source_id, + "edge_type": "direct_tool", + "source": "agent.py", + "source_pointer": "/Agent/tools", + "complete": True, + } + ] + return Tool( + id=f"tool:{source_id}:{name}", + name=name, + provider=source_id, + source_type="mcp", + source_id=source_id, + annotations=annotations, + auth=AuthInfo(mode="none", explicit=True), + extraction_confidence="high", + ) + + +def _manifest(*, bindings: dict | None = None, sdk_object: str | None = None): + agent: dict = { + "name": "test-agent", + "declared_purpose": ["test exact static bindings"], + } + if sdk_object: + agent["sdk"] = {"type": "test", "object": sdk_object} + return AgentsShipgateManifest.model_validate( + { + "version": "0.1", + "project": {"name": "binding-test"}, + "agent": agent, + "environment": {"target": "local"}, + "tool_sources": [{"id": "catalog", "type": "mcp", "path": "tools.json"}], + "agent_bindings": bindings or {}, + } + ) + + +def test_catalog_membership_never_implies_agent_binding() -> None: + graph, tools = resolve_agent_binding_graph( + _manifest(), [_tool("orders.process")], ArtifactBag() + ) + + assert graph.pass_eligible is False + assert graph.reachable_tool_ids == [] + assert graph.unbound_tool_ids == ["tool:catalog:orders.process"] + assert {issue.kind for issue in graph.issues} == {"ambiguous_root_agent"} + assert tools[0].binding_assessment is not None + assert tools[0].binding_assessment.pass_eligible is False + + +def test_structural_binding_selects_only_reachable_tools() -> None: + bound = _tool("orders.lookup", agent="root_agent") + unbound = _tool("orders.delete") + graph, _ = resolve_agent_binding_graph( + _manifest(sdk_object="root_agent"), [bound, unbound], ArtifactBag() + ) + + assert graph.status == "structural" + assert graph.pass_eligible is True + assert graph.reachable_tool_ids == [bound.id] + assert graph.unbound_tool_ids == [unbound.id] + + +def test_exact_reviewed_empty_binding_can_prove_zero_capabilities() -> None: + graph, _ = resolve_agent_binding_graph( + _manifest( + bindings={ + "declarations": [ + { + "agent": "root", + "complete": True, + "tools": [], + "handoffs": [], + "reason": "reviewed empty root surface", + } + ] + } + ), + [], + ArtifactBag(), + ) + + assert graph.status == "declared" + assert graph.pass_eligible is True + assert graph.reachable_tool_ids == [] + + +def test_reviewed_binding_uses_canonical_source_qualified_selector() -> None: + first = _tool("lookup", source_id="provider-a") + second = _tool("lookup", source_id="provider-b") + graph, _ = resolve_agent_binding_graph( + _manifest( + bindings={ + "declarations": [ + { + "agent": "root", + "complete": True, + "tools": [{"tool": "lookup", "source_id": "provider-b"}], + "reason": "reviewed provider-b binding", + } + ] + } + ), + [first, second], + ArtifactBag(), + ) + + assert graph.pass_eligible is True + assert graph.reachable_tool_ids == [second.id] + assert graph.unbound_tool_ids == [first.id] + + +def test_declaration_cannot_erase_complete_structural_edge() -> None: + structural = _tool("orders.delete", source_id="sdk", agent="root_agent") + declared_only = _tool("orders.lookup", source_id="sdk") + graph, _ = resolve_agent_binding_graph( + _manifest( + sdk_object="root_agent", + bindings={ + "root": {"source_id": "sdk", "object": "root_agent"}, + "declarations": [ + { + "agent": "root", + "complete": True, + "tools": [{"tool": "orders.lookup", "source_id": "sdk"}], + "reason": "incorrect reviewed downgrade", + } + ], + }, + ), + [structural, declared_only], + ArtifactBag(), + ) + + assert graph.status == "conflicting" + assert graph.pass_eligible is False + assert "conflicting_binding_evidence" in {issue.kind for issue in graph.issues} + assert structural.id in graph.reachable_tool_ids + + +def test_dynamic_binding_annotation_fails_closed() -> None: + tool = _tool("lookup", agent="root_agent") + tool.annotations["binding_surface_partial"] = ["tools are loaded from runtime config"] + graph, _ = resolve_agent_binding_graph( + _manifest(sdk_object="root_agent"), [tool], ArtifactBag() + ) + + assert graph.status == "partial" + assert graph.pass_eligible is False + assert "partial_binding_evidence" in {issue.kind for issue in graph.issues} diff --git a/tests/test_agent_instructions_apply.py b/tests/test_agent_instructions_apply.py index 99105140..eef4292f 100644 --- a/tests/test_agent_instructions_apply.py +++ b/tests/test_agent_instructions_apply.py @@ -190,7 +190,7 @@ def test_claude_command_current_file_matches_renderer() -> None: def test_local_contract_renderer_has_required_fields() -> None: payload = json.loads(render_local_contract_file()) assert payload["schema_version"] == "2" - assert payload["contract_version"] == "12" + assert payload["contract_version"] == "13" assert "verify_local" not in payload["primary_commands"] assert payload["primary_commands"]["verify_pr"].startswith("agents-shipgate verify") assert payload["commands"]["verify_local"].startswith("agents-shipgate verify") diff --git a/tests/test_agent_instructions_renderers.py b/tests/test_agent_instructions_renderers.py index f202de2f..359be814 100644 --- a/tests/test_agent_instructions_renderers.py +++ b/tests/test_agent_instructions_renderers.py @@ -45,10 +45,10 @@ REPO_ROOT = Path(__file__).resolve().parent.parent EXPECTED_CLAUDE_CODE_SKILL_RENDER_SHA256 = { ".claude/skills/agents-shipgate/SKILL.md": ( - "66e31ffa9b2f0f1b254f4a5d627ca296a2b278043c61d59026c779bb4a1f609b" + "7d2ea9407f248e7eb54113fefe9aac8d9e5dabb4013d1f2327b42442072d0041" ), ".claude/skills/agents-shipgate/ci-recipes/advisory-pr-comment.yml": ( - "b2e42bce1eb6892c5207890bd53ab987d0beaac7c9fa2d9de8ba07033a3f2f60" + "e4c4f9c15af4ad9e537c0f24eff09472b3728e2ce08f865602fd7b1d95f60cb7" ), ".claude/skills/agents-shipgate/prompts/add-shipgate-to-repo.md": ( "b3a3273bf68c3f49abd32585f8ce6e9f562c49be520f8d3f381e39afa4712280" @@ -80,7 +80,7 @@ } EXPECTED_CODEX_SKILL_RENDER_SHA256 = { ".agents/skills/agents-shipgate/SKILL.md": ( - "d2d6549ca3bde2ec2581e8712419ace340afc5be3078f2255884670f4a8db870" + "37795c6ffc3dcdda624b609dd17da8656c245d00ea9cebb9fae25c50842a4a9b" ), ".agents/skills/agents-shipgate/agents/openai.yaml": ( "aa511e933ff663dcd1e0d2af3da2a7101206ce2bb1bb98c4dae801bb3f4e42ef" @@ -165,7 +165,7 @@ def test_local_contract_renderer_exposes_agent_operational_fields() -> None: payload = json.loads(render_local_contract_file()) assert payload["schema_version"] == "2" assert payload["agents_shipgate_version"] - assert payload["contract_version"] == "12" + assert payload["contract_version"] == "13" assert payload["primary_commands"]["verify_pr"].startswith("agents-shipgate verify") assert payload["primary_commands"]["host_audit"].startswith("shipgate audit --host") assert "verify_local" not in payload["primary_commands"] diff --git a/tests/test_agent_mode.py b/tests/test_agent_mode.py index 8a38b9df..1ea4fb93 100644 --- a/tests/test_agent_mode.py +++ b/tests/test_agent_mode.py @@ -228,7 +228,7 @@ def test_verify_json_shortcut_prints_verifier_artifact(tmp_path: Path) -> None: assert result.exit_code == 0, result.output payload = json.loads(result.output) assert payload["verifier_schema_version"] == "0.2" - assert payload["merge_verdict"] == "human_review_required" + assert payload["merge_verdict"] == "insufficient_evidence" assert payload["can_merge_without_human"] is False # Full artifacts still land on disk for the documented file contract. assert (repo / "agents-shipgate-reports" / "verifier.json").is_file() @@ -293,7 +293,7 @@ def test_verify_agent_environment_defaults_to_verifier_json( assert result.exit_code == 0, result.output payload = json.loads(result.output) assert payload["verifier_schema_version"] == "0.2" - assert payload["merge_verdict"] == "human_review_required" + assert payload["merge_verdict"] == "insufficient_evidence" def test_verify_without_agent_environment_defaults_to_text( diff --git a/tests/test_anthropic_api.py b/tests/test_anthropic_api.py index b45df9b5..c21db53d 100644 --- a/tests/test_anthropic_api.py +++ b/tests/test_anthropic_api.py @@ -215,6 +215,13 @@ def test_anthropic_client_tool_fires_high_risk_release_findings(tmp_path): - prompt.md tools: - path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: [{tool: bash}] + handoffs: [] + reason: reviewed Anthropic client-tool fixture binding """, encoding="utf-8", ) diff --git a/tests/test_capability_change_schema_hash_parity.py b/tests/test_capability_change_schema_hash_parity.py index 3a24e4b9..8164c89f 100644 --- a/tests/test_capability_change_schema_hash_parity.py +++ b/tests/test_capability_change_schema_hash_parity.py @@ -71,6 +71,15 @@ - id: catalog-api type: openapi path: openapi.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: list_items, source_id: catalog-api} + - {tool: get_item, source_id: catalog-api} + handoffs: [] + reason: reviewed parity fixture binding """ diff --git a/tests/test_capability_domain.py b/tests/test_capability_domain.py index 5dd542cd..2b1824f6 100644 --- a/tests/test_capability_domain.py +++ b/tests/test_capability_domain.py @@ -389,4 +389,4 @@ def test_building_capability_facts_does_not_change_action_fact_output() -> None: def test_capability_substrate_uses_current_report_schema_version() -> None: - assert ReadinessReport.model_fields["report_schema_version"].default == "0.30" + assert ReadinessReport.model_fields["report_schema_version"].default == "0.31" diff --git a/tests/test_capability_lock.py b/tests/test_capability_lock.py index facaf702..ba202f49 100644 --- a/tests/test_capability_lock.py +++ b/tests/test_capability_lock.py @@ -476,7 +476,7 @@ def test_mixed_capability_standard_diff_requires_exact_reexport( assert str(exc_info.value) == ( "Mixed capability-standard lock diff is not comparable " - "(base=0.2, head=0.3). Re-export the base lock from its source " + "(base=0.2, head=0.4). Re-export the base lock from its source " "workspace with the current engine using exactly: " f"`{command}`. Then rerun the capability diff." ) @@ -569,6 +569,15 @@ def _write_project(path: Path) -> None: - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: cases.search, source_id: tools} + - {tool: cases.update, source_id: tools} + handoffs: [] + reason: reviewed capability-lock fixture binding """, encoding="utf-8", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 73be3ce5..e7efe22b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -798,7 +798,8 @@ def test_cli_scan_workspace_writes_separate_report_dirs(tmp_path): assert decision_lines, "expected per-manifest summary lines in workspace output" for line in decision_lines: assert any( - f": {decision} " in line for decision in ("blocked", "review_required", "passed") + f": {decision} " in line + for decision in ("blocked", "review_required", "insufficient_evidence", "passed") ), f"workspace summary line should lead with decision: {line!r}" assert "blockers=" in line @@ -839,6 +840,22 @@ def test_cli_scan_workspace_continues_after_config_error(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: docs.lookup + source_id: tools + handoffs: [] + reason: reviewed workspace test binding +action_surface: + actions: + - tool: docs.lookup + source_id: tools + effect: read + authority: + mode: none """, encoding="utf-8", ) diff --git a/tests/test_codex_plugin_launch_package.py b/tests/test_codex_plugin_launch_package.py index e151ab8b..0639620c 100644 --- a/tests/test_codex_plugin_launch_package.py +++ b/tests/test_codex_plugin_launch_package.py @@ -90,6 +90,13 @@ def test_agents_shipgate_codex_plugin_scans_without_plugin_findings( type: codex_plugin mode: marketplace path: .agents/plugins/marketplace.json + agent_bindings: + declarations: + - agent: root + complete: true + tools: [] + handoffs: [] + reason: reviewed skill-only plugin has no callable tools output: packet: enabled: false diff --git a/tests/test_crewai.py b/tests/test_crewai.py index 83951943..946e3ca1 100644 --- a/tests/test_crewai.py +++ b/tests/test_crewai.py @@ -296,5 +296,7 @@ def test_crewai_inventory_suppresses_dynamic_surface_finding(tmp_path): assert "SHIP-CREWAI-DYNAMIC-TOOL-SURFACE-NOT-ENUMERABLE" not in { finding.check_id for finding in report.findings } - assert report.tool_inventory[0]["source_type"] == "crewai_inventory" + assert report.tool_catalog[0]["source_type"] == "crewai_inventory" + assert report.tool_inventory == [] + assert report.binding_surface_facts.status == "partial" assert doctor["frameworks"]["crewai"]["tool_inventory_file_count"] == 1 diff --git a/tests/test_evidence_backed_pass.py b/tests/test_evidence_backed_pass.py index 74d964d8..bedb9b46 100644 --- a/tests/test_evidence_backed_pass.py +++ b/tests/test_evidence_backed_pass.py @@ -66,6 +66,16 @@ def _write_project( checks_block = "\nchecks:\n ignore:\n" + "\n".join(rows) + "\n" manifest = root / "shipgate.yaml" + binding_lines = [ + "agent_bindings:", + " declarations:", + " - agent: root", + " complete: true", + " tools:", + *[f" - {{tool: {tool['name']}, source_id: tools}}" for tool in tools], + " handoffs: []", + " reason: reviewed P0 semantic fixture binding", + ] manifest.write_text( """version: "0.1" project: @@ -85,6 +95,8 @@ def _write_project( directory: reports formats: [json] """ + + "\n".join(binding_lines) + + "\n" + action_surface + policy_block + checks_block, diff --git a/tests/test_evidence_packet.py b/tests/test_evidence_packet.py index 97b2ea8a..41ae1a78 100644 --- a/tests/test_evidence_packet.py +++ b/tests/test_evidence_packet.py @@ -148,7 +148,7 @@ def test_packet_emits_alongside_report_by_default(tmp_path): out, packet = _scan_with_packet(tmp_path) for name in ("packet.md", "packet.json", "packet.html"): assert (out / name).exists(), name - assert packet.packet_schema_version == "0.9" + assert packet.packet_schema_version == "0.10" def test_packet_v07_keeps_capability_refs_on_release_items(): @@ -935,7 +935,7 @@ def test_load_packet_json_upgrades_v02_hitl_fields(tmp_path): upgraded = load_packet_json(payload) - assert upgraded.packet_schema_version == "0.9" + assert upgraded.packet_schema_version == "0.10" assert upgraded.evidence_matrix.notes assert upgraded.action_surface_diff.status == "not_declared" assert upgraded.action_surface_diff.enabled is False @@ -958,7 +958,7 @@ def test_load_packet_json_upgrades_v01_to_v05(tmp_path): upgraded = load_packet_json(payload) - assert upgraded.packet_schema_version == "0.9" + assert upgraded.packet_schema_version == "0.10" assert upgraded.evidence_matrix.notes assert upgraded.tool_surface_diff.status == "not_declared" assert upgraded.tool_surface_diff.enabled is False @@ -979,7 +979,7 @@ def test_load_packet_json_upgrades_v04_action_surface_section(tmp_path): upgraded = load_packet_json(payload) - assert upgraded.packet_schema_version == "0.9" + assert upgraded.packet_schema_version == "0.10" assert upgraded.action_surface_diff.status == "not_declared" assert upgraded.action_surface_diff.enabled is False assert upgraded.evidence_matrix.notes @@ -993,7 +993,7 @@ def test_load_packet_json_upgrades_v05_evidence_matrix(tmp_path): upgraded = load_packet_json(payload) - assert upgraded.packet_schema_version == "0.9" + assert upgraded.packet_schema_version == "0.10" assert len(upgraded.evidence_matrix.rows) == 13 assert any("older packet schema" in note for note in upgraded.evidence_matrix.notes) @@ -1034,7 +1034,7 @@ def test_legacy_passed_packet_downgrades_to_actionable_insufficient_evidence( upgraded = load_packet_json(payload) - assert upgraded.packet_schema_version == "0.9" + assert upgraded.packet_schema_version == "0.10" assert upgraded.release_decision.decision == "insufficient_evidence" assert upgraded.release_decision.verdict == "INSUFFICIENT EVIDENCE" assert "not a v0.8 safety statement" in upgraded.release_decision.reason @@ -1270,7 +1270,7 @@ def test_evidence_packet_writes_packet_json_when_format_includes_json(tmp_path): # The written packet.json must round-trip. payload = (target / "packet.json").read_text(encoding="utf-8") reloaded = load_packet_json(payload) - assert reloaded.packet_schema_version == "0.9" + assert reloaded.packet_schema_version == "0.10" def test_evidence_packet_pdf_only_exits_zero_when_weasyprint_missing(tmp_path, monkeypatch): @@ -1397,7 +1397,7 @@ def test_evidence_packet_cli_round_trips(tmp_path): ) assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["packet_schema_version"] == "0.9" + assert payload["packet_schema_version"] == "0.10" assert payload["run_id"] == packet.run_id diff --git a/tests/test_fingerprint_compatibility.py b/tests/test_fingerprint_compatibility.py index 8324d8e8..1a653040 100644 --- a/tests/test_fingerprint_compatibility.py +++ b/tests/test_fingerprint_compatibility.py @@ -10,7 +10,6 @@ from agents_shipgate.schemas.report import Finding V015_FINGERPRINTS = { - ("SHIP-INVENTORY-WILDCARD-TOOLS", "wildcard_mcp_tools.*"): "fp_fc02d8ecd30f2578", ("SHIP-SCHEMA-MISSING-BOUNDS", "stripe.create_refund"): "fp_ab60b01cb53cfcbe", ("SHIP-SCHEMA-BROAD-FREE-TEXT", "zendesk.update_ticket"): "fp_ff2f028953d1c220", ("SHIP-SCHEMA-BROAD-FREE-TEXT", "gmail.send_customer_email"): "fp_acd63b899d49aa1c", @@ -37,6 +36,8 @@ def test_v015_unchanged_finding_fingerprints_remain_stable(tmp_path: Path) -> No for finding in report.findings } assert {key: observed.get(key) for key in V015_FINGERPRINTS} == V015_FINGERPRINTS + assert any(row["name"] == "wildcard_mcp_tools.*" for row in report.tool_catalog) + assert all(row["name"] != "wildcard_mcp_tools.*" for row in report.tool_inventory) def test_v015_freeform_output_fingerprint_remains_stable_after_sample_migration() -> None: diff --git a/tests/test_fixture_no_import.py b/tests/test_fixture_no_import.py index fa93b82c..9a3850db 100644 --- a/tests/test_fixture_no_import.py +++ b/tests/test_fixture_no_import.py @@ -106,10 +106,10 @@ def _run_and_assert_no_import(workspace: Path) -> object: def _assert_did_work_tool_inventory(report: object, *, minimum: int = 1) -> None: - """For adapters that surface tools into ``report.tool_inventory``.""" - count = len(report.tool_inventory) # type: ignore[attr-defined] + """For adapters that surface declarations into ``report.tool_catalog``.""" + count = len(report.tool_catalog) # type: ignore[attr-defined] assert count >= minimum, ( - f"Adapter did not extract enough tools into tool_inventory " + f"Adapter did not extract enough tools into tool_catalog " f"(got {count}, expected ≥ {minimum}). If the adapter changed its " "scope, update the fixture or the assertion." ) diff --git a/tests/test_google_adk.py b/tests/test_google_adk.py index 12bd9b48..1b760b39 100644 --- a/tests/test_google_adk.py +++ b/tests/test_google_adk.py @@ -289,7 +289,8 @@ def test_google_adk_top_level_config_can_supply_inputs(tmp_path): ci_mode="advisory", ) - assert report.tool_inventory[0]["name"] == "support.lookup" + assert report.tool_catalog[0]["name"] == "support.lookup" + assert report.tool_inventory == [] assert report.frameworks["google_adk"]["tool_inventory_file_count"] == 1 assert "SHIP-ADK-EVAL-COVERAGE-MISSING" not in { finding.check_id for finding in report.findings diff --git a/tests/test_governance_benchmark.py b/tests/test_governance_benchmark.py index 8bc81875..419876fc 100644 --- a/tests/test_governance_benchmark.py +++ b/tests/test_governance_benchmark.py @@ -79,7 +79,7 @@ def test_runner_executes_selected_case_deterministically(tmp_path: Path) -> None assert first.summary.selected_cases == 1 assert first.summary.passed_cases == 1 assert first.cases[0].actual is not None - assert first.cases[0].actual.merge_verdict == "mergeable" + assert first.cases[0].actual.merge_verdict == "human_review_required" assert first.cases[0].capability_diff is not None assert first.cases[0].capability_diff.added == 1 @@ -97,11 +97,11 @@ def test_full_executable_governance_benchmark_passes(tmp_path: Path) -> None: assert result.summary.selected_cases == 10 assert result.summary.failed_cases == 0 assert {metric.metric: metric.total for metric in result.metrics} == { - "authority_routing": 6, + "authority_routing": 8, "capability_semantic_fidelity": 6, "explanation_usefulness": 10, "remediation_boundary": 6, - "safe_pass_rate": 4, + "safe_pass_rate": 2, "unsafe_merge_prevention": 4, } @@ -227,7 +227,7 @@ def test_governance_benchmark_schemas_validate_catalog_and_result(tmp_path: Path def test_governance_benchmark_preserves_public_schema_boundaries() -> None: - assert ReadinessReport.model_fields["report_schema_version"].default == "0.30" - assert CAPABILITY_LOCK_SCHEMA_VERSION == "0.4" - assert CAPABILITY_LOCK_DIFF_SCHEMA_VERSION == "0.5" + assert ReadinessReport.model_fields["report_schema_version"].default == "0.31" + assert CAPABILITY_LOCK_SCHEMA_VERSION == "0.5" + assert CAPABILITY_LOCK_DIFF_SCHEMA_VERSION == "0.6" assert GOVERNANCE_BENCHMARK_RESULT_SCHEMA_VERSION == "0.2" diff --git a/tests/test_langchain.py b/tests/test_langchain.py index 96189e48..9607830b 100644 --- a/tests/test_langchain.py +++ b/tests/test_langchain.py @@ -224,10 +224,13 @@ def summarize_case_wrapped(case_id: str) -> dict: ci_mode="advisory", ) - assert {tool["name"] for tool in report.tool_inventory} == { + assert {tool["name"] for tool in report.tool_catalog} == { "summarize_case", "summarize_case_wrapped", } + assert {tool["name"] for tool in report.tool_inventory} == { + "summarize_case_wrapped" + } assert report.frameworks["langchain"]["dynamic_tool_surface_count"] == 1 assert any("tool variable 'summarize_case' is reassigned" in w for w in report.source_warnings) @@ -282,5 +285,7 @@ def test_langchain_inventory_suppresses_dynamic_surface_finding(tmp_path): assert "SHIP-LANGCHAIN-DYNAMIC-TOOL-SURFACE-NOT-ENUMERABLE" not in { finding.check_id for finding in report.findings } - assert report.tool_inventory[0]["source_type"] == "langchain_inventory" + assert report.tool_catalog[0]["source_type"] == "langchain_inventory" + assert report.tool_inventory == [] + assert report.binding_surface_facts.status == "partial" assert doctor["frameworks"]["langchain"]["tool_inventory_file_count"] == 1 diff --git a/tests/test_large_sample.py b/tests/test_large_sample.py index 4cf4e8ef..1bf1dfa7 100644 --- a/tests/test_large_sample.py +++ b/tests/test_large_sample.py @@ -52,12 +52,11 @@ # (a new check landing, a slightly tightened risk classifier) without # false-failing, tight enough to catch a regression that drops half the # pipeline. -MIN_LOADED_TOOLS = 50 -MAX_LOADED_TOOLS = 100 -MIN_TOTAL_FINDINGS = 40 -MAX_TOTAL_FINDINGS = 200 -MIN_CRITICAL_FINDINGS = 3 # at minimum, the financial-action approval gaps -MIN_BLOCKERS = 3 +MIN_CATALOG_TOOLS = 50 +MAX_CATALOG_TOOLS = 100 +EXPECTED_REACHABLE_TOOLS = 5 +MIN_TOTAL_FINDINGS = 20 +MAX_TOTAL_FINDINGS = 60 MIN_REVIEW_ITEMS = 20 # Six tool sources are declared; the reviewed inventory intentionally outranks # the five duplicate SDK tools while the merge receipts prove the AST adapter @@ -131,7 +130,7 @@ def test_all_six_tool_sources_load(scanned_sample: ReadinessReport) -> None: # prefix. The reviewed inventory outranks duplicate SDK facts in the final # inventory, so the SDK contribution is asserted through merge receipts. seen_files: set[str] = set() - for entry in scanned_sample.tool_inventory: + for entry in scanned_sample.tool_catalog: ref = entry.get("source_ref") or "" if "#" in ref: ref = ref.split("#", 1)[0] @@ -152,14 +151,14 @@ def test_all_six_tool_sources_load(scanned_sample: ReadinessReport) -> None: # Also pin the per-adapter cohort: at least one tool from each loader # surface type. This catches a different regression — a source still # loading but emitting tools with the wrong source_type tag. - source_types = {entry.get("source_type") for entry in scanned_sample.tool_inventory} + source_types = {entry.get("source_type") for entry in scanned_sample.tool_catalog} assert {"openapi", "mcp"}.issubset(source_types), ( f"Expected source_types to include openapi + mcp; " f"got {sorted(source_types)}." ) sdk_bindings = [ row - for row in scanned_sample.tool_inventory + for row in scanned_sample.tool_catalog if row.get("provider") == "ops_sdk" and len(row.get("observation_ids", [])) == 2 ] @@ -167,11 +166,11 @@ def test_all_six_tool_sources_load(scanned_sample: ReadinessReport) -> None: def test_tool_inventory_is_in_expected_band(scanned_sample: ReadinessReport) -> None: - n = len(scanned_sample.tool_inventory) - assert MIN_LOADED_TOOLS <= n <= MAX_LOADED_TOOLS, ( - f"Expected {MIN_LOADED_TOOLS}–{MAX_LOADED_TOOLS} loaded tools; got " - f"{n}. A change outside that band signals either a sample edit or an " - "adapter regression." + assert len(scanned_sample.tool_inventory) == EXPECTED_REACHABLE_TOOLS + n = len(scanned_sample.tool_catalog) + assert MIN_CATALOG_TOOLS <= n <= MAX_CATALOG_TOOLS + assert scanned_sample.binding_surface_facts.reachable_tool_ids == sorted( + tool["tool_id"] for tool in scanned_sample.tool_inventory ) @@ -184,20 +183,16 @@ def test_findings_count_is_in_expected_band(scanned_sample: ReadinessReport) -> ) -def test_release_decision_is_blocked(scanned_sample: ReadinessReport) -> None: - """The sample is deliberately authored with critical approval gaps.""" +def test_release_decision_requires_review(scanned_sample: ReadinessReport) -> None: + """Only the five statically wired local SDK helpers are in scope.""" decision = scanned_sample.release_decision assert decision is not None - assert decision.decision == "blocked", ( - f"Expected decision='blocked'; got {decision.decision!r}. If the " - "sample was edited to be cleaner, update this test and document the " - "intent." - ) - assert len(decision.blockers) >= MIN_BLOCKERS + assert decision.decision == "review_required" + assert not decision.blockers assert len(decision.review_items) >= MIN_REVIEW_ITEMS -def test_at_least_one_critical_approval_gap_fires( +def test_unbound_financial_catalog_entries_do_not_create_control_findings( scanned_sample: ReadinessReport, ) -> None: """Anchor the headline check we authored the sample around.""" @@ -208,23 +203,20 @@ def test_at_least_one_critical_approval_gap_fires( and f.check_id == "SHIP-POLICY-APPROVAL-MISSING" and not f.suppressed ] - assert len(critical_approval_findings) >= MIN_CRITICAL_FINDINGS, ( - "Expected the financial-action tools without declared approval to " - "fire SHIP-POLICY-APPROVAL-MISSING at critical severity. Got " - f"{len(critical_approval_findings)} critical approval findings." - ) + assert critical_approval_findings == [] + assert len(scanned_sample.binding_surface_facts.unbound_tool_ids) == 58 -def test_scope_coverage_check_fires(scanned_sample: ReadinessReport) -> None: - """The manifest deliberately omits several admin scopes.""" +def test_unbound_scopes_are_reported_as_stale_not_capability_gaps(scanned_sample: ReadinessReport) -> None: scope_findings = [ f for f in scanned_sample.findings if f.check_id == "SHIP-AUTH-SCOPE-COVERAGE-MISSING" and not f.suppressed ] - assert scope_findings, ( - "Expected SHIP-AUTH-SCOPE-COVERAGE-MISSING to fire for the admin " - "scopes the manifest omits." + assert scope_findings == [] + assert any( + finding.check_id == "SHIP-MANIFEST-UNUSED-SCOPE" + for finding in scanned_sample.findings ) @@ -278,7 +270,7 @@ def test_reviewer_summary_block_is_populated( appear on every emitted scan.""" summary = scanned_sample.reviewer_summary assert summary is not None - assert summary.verdict == "blocked" + assert summary.verdict == "review_required" assert summary.headline # non-empty diff --git a/tests/test_latency_budget.py b/tests/test_latency_budget.py index 82391686..a99217d8 100644 --- a/tests/test_latency_budget.py +++ b/tests/test_latency_budget.py @@ -101,7 +101,7 @@ def _measure_one(config_path: Path, output_dir: Path) -> tuple[float, int]: packet_enabled=False, # packet PDF is the optional extra; not part of the gate. plugins_enabled=False, # plugins are off by default for perf budget. ) - return time.perf_counter() - start, len(report.tool_inventory) + return time.perf_counter() - start, len(report.tool_catalog) def _phase_breakdown_lines() -> list[str]: @@ -232,7 +232,7 @@ def test_scenarios_scale_sublinearly(perf_session: None, tmp_path: Path) -> None packet_enabled=False, plugins_enabled=False, ) - tool_counts[size] = len(report.tool_inventory) + tool_counts[size] = len(report.tool_catalog) tool_ratio = tool_counts["large"] / max(tool_counts["small"], 1) latency_ratio = medians["large"] / max(medians["small"], 0.001) diff --git a/tests/test_manifest_consistency.py b/tests/test_manifest_consistency.py index dadac5e8..280585be 100644 --- a/tests/test_manifest_consistency.py +++ b/tests/test_manifest_consistency.py @@ -39,6 +39,13 @@ def test_manifest_consistency_flags_stale_entries_and_unused_scopes(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: [{tool: billing.create_refund, source_id: tools}] + handoffs: [] + reason: reviewed manifest-consistency fixture binding permissions: scopes: - billing:* @@ -107,6 +114,13 @@ def test_manifest_consistency_has_no_false_positive_for_current_entries(tmp_path - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: [{tool: support.lookup, source_id: tools}] + handoffs: [] + reason: reviewed manifest-consistency fixture binding permissions: scopes: - support:read diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index e1802b56..ffd69689 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -135,7 +135,7 @@ def test_mcp_capabilities_handler_does_not_write_reports(tmp_path: Path) -> None no_plugins=True, ) - assert payload["capability_lock_schema_version"] == "0.4" + assert payload["capability_lock_schema_version"] == "0.5" assert _snapshot(workspace) == before diff --git a/tests/test_miner_constructed.py b/tests/test_miner_constructed.py index 2d3262b5..8ec7bb5f 100644 --- a/tests/test_miner_constructed.py +++ b/tests/test_miner_constructed.py @@ -90,8 +90,6 @@ def test_ie_threshold_is_exercised_and_robust_on_the_labeled_coverage_fixture( the constant. Freezing the constant itself is ``test_ie_threshold_constants_are_frozen`` in ``test_release_decision.py``. """ - from agents_shipgate.ci.release_decision import _low_confidence_tool_threshold - project = tmp_path / "ambiguous-sdk" project.mkdir() (project / "agent.py").write_text( @@ -151,17 +149,13 @@ def inspect_record(record_id: str) -> dict: report = json.loads(report_path.read_text(encoding="utf-8")) decision = report["release_decision"] coverage = decision["evidence_coverage"] - low = coverage["low_confidence_tool_count"] - total = (report.get("tool_surface") or {}).get("total_tools") or len( - report.get("tool_inventory") or [] - ) assert decision["decision"] == "insufficient_evidence" - # The current constant correctly classifies it as below-threshold. - assert low >= _low_confidence_tool_threshold(total) - # It sits at the robust extreme (every tool is low-confidence), so this - # single labeled point cannot distinguish ratio values in (0, 1]. Moving - # the constant needs the human labeling pass + a re-mine, not this fixture. - assert low == total and total > 0 + assert len(report["tool_catalog"]) == 1 + assert report["tool_inventory"] == [] + binding = coverage["binding_coverage"] + assert binding["reachable_tools"] == 0 + assert binding["unbound_tools"] == 1 + assert binding["gap_count"] >= 1 def test_cli_env_prepends_this_checkouts_src() -> None: diff --git a/tests/test_n8n.py b/tests/test_n8n.py index 5dbb664b..88a8db41 100644 --- a/tests/test_n8n.py +++ b/tests/test_n8n.py @@ -223,12 +223,9 @@ def test_n8n_mcp_server_trigger_exposed_tools_are_normalized_as_mcp(tmp_path): frameworks = report.frameworks["n8n"] assert frameworks["mcp_server_trigger_count"] == 1 assert frameworks["mcp_server_exposed_tool_count"] == 1 - tool = next(item for item in report.tool_inventory if item["name"] == "Lookup Case") + tool = next(item for item in report.tool_catalog if item["name"] == "Lookup Case") assert tool["source_type"] == "mcp" - loaded_tool = next( - item for item in report.tool_surface_facts.tools if item.name == "Lookup Case" - ) - assert loaded_tool.source_type == "mcp" + assert report.tool_inventory == [] def test_n8n_directory_mode_filters_non_workflows_and_is_deterministic(tmp_path): @@ -264,7 +261,7 @@ def test_n8n_directory_mode_filters_non_workflows_and_is_deterministic(tmp_path) assert report.frameworks["n8n"]["workflow_file_count"] == 2 assert [ - tool["name"] for tool in report.tool_inventory if tool["name"] in {"A Tool", "B Tool"} + tool["name"] for tool in report.tool_catalog if tool["name"] in {"A Tool", "B Tool"} ] == [ "A Tool", "B Tool", @@ -531,22 +528,12 @@ def test_n8n_downstream_send_and_wait_is_not_approval_evidence(tmp_path): ci_mode="advisory", ) - shared_tools = [ - tool for tool in report.tool_surface_facts.tools if tool.name == "shared_delete" - ] + shared_tools = [tool for tool in report.tool_catalog if tool["name"] == "shared_delete"] assert len(shared_tools) == 2 assert report.frameworks["n8n"]["human_review_node_count"] == 1 - findings = [ - finding - for finding in report.findings - if finding.check_id == "SHIP-POLICY-APPROVAL-MISSING" - and finding.tool_name == "shared_delete" - ] - assert len(findings) == 2 - assert {finding.source.ref for finding in findings if finding.source is not None} == { - "workflows/a-reviewed.json#node:reviewed-tool", - "workflows/b-unreviewed.json#node:unreviewed-tool", - } + assert report.binding_surface_facts.status == "unknown" + assert report.tool_inventory == [] + assert not [finding for finding in report.findings if finding.tool_name == "shared_delete"] def test_n8n_regular_code_node_bound_as_ai_tool_gets_code_execution_hint(tmp_path): @@ -828,7 +815,7 @@ def test_n8n_same_named_tools_in_different_workflows_are_not_deduped(tmp_path): ) inventory_entries = [ - tool for tool in report.tool_inventory if tool["name"] == "lookup_customer" + tool for tool in report.tool_catalog if tool["name"] == "lookup_customer" ] assert len(inventory_entries) == 2 assert {tool["source_ref"] for tool in inventory_entries} == { @@ -871,7 +858,7 @@ def test_n8n_array_export_file_loads_multiple_workflows(tmp_path): ) assert report.frameworks["n8n"]["workflow_count"] == 2 - inventory = {tool["name"]: tool for tool in report.tool_inventory} + inventory = {tool["name"]: tool for tool in report.tool_catalog} assert inventory["alpha_tool"]["source_ref"] == "bundle.json#node:a-tool" assert inventory["beta_tool"]["source_ref"] == "bundle.json#node:b-tool" @@ -1138,7 +1125,7 @@ def test_n8n_multi_bound_tool_keeps_agent_and_mcp_surfaces(tmp_path): ci_mode="advisory", ) - entries = [tool for tool in report.tool_inventory if tool["name"] == "Dual Tool"] + entries = [tool for tool in report.tool_catalog if tool["name"] == "Dual Tool"] assert {tool["source_type"] for tool in entries} == {"mcp", "n8n_workflow_tool"} assert report.frameworks["n8n"]["ai_agent_count"] == 2 assert report.frameworks["n8n"]["mcp_server_trigger_count"] == 1 diff --git a/tests/test_openai_api.py b/tests/test_openai_api.py index eb0f239f..6116a3d4 100644 --- a/tests/test_openai_api.py +++ b/tests/test_openai_api.py @@ -236,8 +236,9 @@ def test_openai_api_source_wins_duplicate_tool_names(tmp_path): ci_mode="advisory", ) - assert report.tool_surface.sources == {"openai_api": 1, "openapi": 1} - assert len({row["tool_id"] for row in report.tool_inventory}) == 2 + assert report.tool_surface.sources == {} + assert {row["provider"] for row in report.tool_catalog} == {"openai_api", "openapi"} + assert len({row["tool_id"] for row in report.tool_catalog}) == 2 assert not any("Duplicate tool name" in warning for warning in report.source_warnings) diff --git a/tests/test_p0_binding_canaries.py b/tests/test_p0_binding_canaries.py new file mode 100644 index 00000000..41a8e7a6 --- /dev/null +++ b/tests/test_p0_binding_canaries.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import pytest + +from agents_shipgate.core.agent_bindings import resolve_agent_binding_graph +from agents_shipgate.core.artifacts import ArtifactBag +from agents_shipgate.core.domain import Tool +from agents_shipgate.schemas.manifest import AgentsShipgateManifest + + +def _manifest(binding_mode: str) -> AgentsShipgateManifest: + payload: dict = { + "version": "0.1", + "project": {"name": "binding-canary"}, + "agent": { + "name": "canary-agent", + "declared_purpose": ["exercise binding canaries"], + }, + "environment": {"target": "local"}, + "tool_sources": [{"id": "source", "type": "mcp", "path": "tools.json"}], + } + if binding_mode == "structural": + payload["agent"]["sdk"] = {"type": "test", "object": "root_agent"} + elif binding_mode == "declared": + payload["agent_bindings"] = { + "declarations": [ + { + "agent": "root", + "complete": True, + "tools": [{"tool": "tool-0", "source_id": "source"}], + "handoffs": [], + "reason": "reviewed canary declaration", + } + ] + } + elif binding_mode == "ambiguous": + payload["agent"]["sdk"] = {"type": "test", "object": "missing_root"} + elif binding_mode == "conflicting": + payload["agent"]["sdk"] = {"type": "test", "object": "root_agent"} + payload["agent_bindings"] = { + "root": {"source_id": "source", "object": "root_agent"}, + "declarations": [ + { + "agent": "root", + "complete": True, + "tools": [], + "handoffs": [], + "reason": "conflicting empty declaration", + } + ], + } + return AgentsShipgateManifest.model_validate(payload) + + +def _tools(count: int, mode: str) -> list[Tool]: + result: list[Tool] = [] + for index in range(count): + annotations: dict = {"readOnlyHint": True} + if mode in {"structural", "partial", "conflicting"} and index == 0: + annotations["agent_bindings"] = [ + { + "agent": "root_agent", + "source_id": "source", + "edge_type": "direct_tool", + "source": "agent.py", + "complete": True, + } + ] + if mode == "partial" and index == 0: + annotations["binding_surface_partial"] = ["runtime-computed tool list"] + result.append( + Tool( + id=f"tool-{index}", + name=f"tool-{index}", + provider="source", + source_type="mcp", + source_id="source", + annotations=annotations, + extraction_confidence="high", + ) + ) + return result + + +CANARIES = [ + *((f"catalog-{index}", "catalog", index + 1, "unknown", False, 0) for index in range(12)), + *((f"structural-{index}", "structural", index + 1, "structural", True, 1) for index in range(10)), + *((f"declared-{index}", "declared", index + 1, "declared", True, 1) for index in range(8)), + *((f"ambiguous-{index}", "ambiguous", index + 1, "unknown", False, 0) for index in range(6)), + *((f"partial-{index}", "partial", index + 1, "partial", False, 1) for index in range(6)), + *((f"conflict-{index}", "conflicting", index + 1, "conflicting", False, 1) for index in range(6)), +] + + +@pytest.mark.parametrize( + ("case_id", "mode", "tool_count", "status", "pass_eligible", "reachable_count"), + CANARIES, + ids=[case[0] for case in CANARIES], +) +def test_48_binding_canaries_have_exact_outcomes( + case_id: str, + mode: str, + tool_count: int, + status: str, + pass_eligible: bool, + reachable_count: int, +) -> None: + del case_id + manifest_mode = "structural" if mode == "partial" else mode + graph, _ = resolve_agent_binding_graph( + _manifest(manifest_mode), + _tools(tool_count, mode), + ArtifactBag(), + ) + + assert graph.status == status + assert graph.pass_eligible is pass_eligible + assert len(graph.reachable_tool_ids) == reachable_count diff --git a/tests/test_p0_safety_canaries.py b/tests/test_p0_safety_canaries.py index 60820144..7999c449 100644 --- a/tests/test_p0_safety_canaries.py +++ b/tests/test_p0_safety_canaries.py @@ -597,6 +597,20 @@ def _write_scan_project( }, "environment": {"target": "production_like"}, "tool_sources": [{"id": "tools", "type": "mcp", "path": "tools.json"}], + "agent_bindings": { + "declarations": [ + { + "agent": "root", + "complete": True, + "tools": [ + {"tool": str(tool["name"]), "source_id": "tools"} + for tool in tools + ], + "handoffs": [], + "reason": "reviewed exact P0 semantic canary surface", + } + ] + }, "ci": {"mode": "advisory"}, "output": {"directory": "reports", "formats": ["json"]}, } diff --git a/tests/test_policy_packs.py b/tests/test_policy_packs.py index d279ae55..5d047bdd 100644 --- a/tests/test_policy_packs.py +++ b/tests/test_policy_packs.py @@ -6,13 +6,41 @@ from typer.testing import CliRunner from agents_shipgate.cli.main import app -from agents_shipgate.cli.scan import run_scan +from agents_shipgate.cli.scan import run_scan as _run_scan from agents_shipgate.core.errors import ConfigError from agents_shipgate.core.findings.identity import legacy_policy_routing_fingerprint from agents_shipgate.schemas.baseline import BaselineFile, BaselineFinding runner = CliRunner() + +def run_scan(*args, **kwargs): + """Keep policy-pack fixtures focused on policy evaluation, not binding gaps.""" + + config_path = Path(kwargs["config_path"]) + text = config_path.read_text(encoding="utf-8") + if ( + "agent_bindings:" not in text + and "type: openapi" in text + and "path: openapi.yaml" in text + and (config_path.parent / "openapi.yaml").is_file() + ): + config_path.write_text( + text + + """ +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: create_refund, source_id: api} + handoffs: [] + reason: reviewed policy-pack fixture binding +""", + encoding="utf-8", + ) + return _run_scan(*args, **kwargs) + ROUTING_EVIDENCE_KEYS = { "policy_owner", "policy_reviewers", @@ -880,6 +908,14 @@ def _manifest_without_policy_pack() -> str: - id: api type: openapi path: openapi.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: create_refund, source_id: api} + handoffs: [] + reason: reviewed policy-pack fixture binding """ diff --git a/tests/test_privacy.py b/tests/test_privacy.py index 266b7df5..29917353 100644 --- a/tests/test_privacy.py +++ b/tests/test_privacy.py @@ -284,6 +284,15 @@ def test_scan_redacts_public_outputs_and_reports_privacy_audit(tmp_path, monkeyp type: openapi path: openapi.yaml optional: false +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: refund_{raw_secret} + source_id: openapi + handoffs: [] + reason: reviewed privacy fixture binding """, encoding="utf-8", ) @@ -398,6 +407,17 @@ def test_scan_does_not_revalidate_manifest_after_redaction_collapses_action_tool type: openapi path: openapi.yaml optional: false +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: "send_{raw_one}" + source_id: openapi + - tool: "send_{raw_two}" + source_id: openapi + handoffs: [] + reason: reviewed privacy fixture bindings """, encoding="utf-8", ) diff --git a/tests/test_release_decision.py b/tests/test_release_decision.py index f0c88ea7..d2be253e 100644 --- a/tests/test_release_decision.py +++ b/tests/test_release_decision.py @@ -19,6 +19,7 @@ Tool, ToolSemanticAssessment, ) +from agents_shipgate.schemas.bindings import AgentBindingGraphAssessment from agents_shipgate.schemas.report import ( BaselineSummary, Finding, @@ -35,6 +36,7 @@ def _finding( baseline_status: str | None = None, requires_human_review: bool | None = None, suppressed: bool = False, + tool_id: str | None = None, ) -> Finding: return Finding( id=f"id-{check_id}-{severity}-{baseline_status or 'new'}", @@ -47,6 +49,7 @@ def _finding( suppressed=suppressed, baseline_status=baseline_status, requires_human_review=requires_human_review, + tool_id=tool_id, ) @@ -102,6 +105,12 @@ def _report( evidence_coverage=evidence_coverage, ), tool_surface=ToolSurfaceSummary(total_tools=len(tools), high_risk_tools=0), + binding_surface_facts=AgentBindingGraphAssessment( + root_agent_id="unit-test-agent", + status="structural", + reachable_tool_ids=[tool.id for tool in tools], + pass_eligible=True, + ), baseline=baseline, findings=findings, source_warnings=source_warnings or [], @@ -420,6 +429,7 @@ def test_active_high_review_finding_outranks_insufficient_evidence(): check_id="SHIP-SCOPE-TOOLKIT-UNBOUNDED", severity="high", baseline_status="new", + tool_id=tools[0].id, ) ], tools=tools, diff --git a/tests/test_reports.py b/tests/test_reports.py index 4f928925..3a9d8c5f 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1213,6 +1213,14 @@ def test_clean_report_has_affirmative_pass_result(tmp_path): - id: docs type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: docs.lookup, source_id: docs} + handoffs: [] + reason: reviewed clean fixture binding action_surface: actions: - tool: docs.lookup diff --git a/tests/test_safety_qualification.py b/tests/test_safety_qualification.py index 1b14aa0a..67b9a29f 100644 --- a/tests/test_safety_qualification.py +++ b/tests/test_safety_qualification.py @@ -41,7 +41,7 @@ ) DECISIONS = ("passed", "review_required", "insufficient_evidence", "blocked") -VERSION = "0.16.0b1" +VERSION = "0.16.0b2" def _human_label(role: str, reviewer: str, decision: str) -> IndependentHumanLabelV1: @@ -104,7 +104,7 @@ def _test_requirements() -> SafetyQualificationRequirementsV1: minimum_blocked_exact=1, minimum_review_exact=1, minimum_insufficient_evidence_exact=1, - required_report_schema_version="0.30", + required_report_schema_version="0.31", ) @@ -116,8 +116,8 @@ def _write_json(path: Path, value: object) -> None: def _write_wheel(path: Path) -> None: with zipfile.ZipFile(path, "w") as archive: archive.writestr( - "agents_shipgate-0.16.0b1.dist-info/METADATA", - "Metadata-Version: 2.4\nName: agents-shipgate\nVersion: 0.16.0b1\n", + "agents_shipgate-0.16.0b2.dist-info/METADATA", + "Metadata-Version: 2.4\nName: agents-shipgate\nVersion: 0.16.0b2\n", ) @@ -161,7 +161,7 @@ def _fixture( actual_overrides: dict[str, str] | None = None, disagreement_case: str | None = None, ) -> tuple[Path, Path, Path, Path]: - wheel = tmp_path / "agents_shipgate-0.16.0b1-py3-none-any.whl" + wheel = tmp_path / "agents_shipgate-0.16.0b2-py3-none-any.whl" _write_wheel(wheel) policy = tmp_path / "qualification-policy.json" _write_json(policy, {"policy": "beta-exact", "version": 1}) @@ -209,6 +209,15 @@ def _fixture( "source_warning_count": 0, "low_confidence_tool_count": 0, "semantic_coverage": _semantic_coverage(actual), + "binding_coverage": { + "total_catalog_tools": 1, + "reachable_tools": 1, + "possible_tools": 0, + "unbound_tools": 0, + "pass_eligible": True, + "gap_count": 0, + "reason_counts": {}, + }, }, "baseline_delta": {"enabled": False}, "fail_policy": { @@ -329,7 +338,7 @@ def test_production_defaults_pin_the_exact_beta_contract() -> None: (item.profile, item.expected_decision): item.count for item in requirements.required_strata } - assert len(counts) == 16 + assert len(counts) == 28 assert sum(counts.values()) == 100 assert sum(count for (_, decision), count in counts.items() if decision == "passed") == 30 assert sum(count for (_, decision), count in counts.items() if decision != "passed") == 70 diff --git a/tests/test_safety_qualification_release.py b/tests/test_safety_qualification_release.py index fa69b450..abfa4959 100644 --- a/tests/test_safety_qualification_release.py +++ b/tests/test_safety_qualification_release.py @@ -26,15 +26,15 @@ verify_release_qualification, ) -VERSION = "0.16.0b1" +VERSION = "0.16.0b2" REPO_ROOT = Path(__file__).resolve().parent.parent def _write_wheel(path: Path) -> None: with zipfile.ZipFile(path, "w") as archive: archive.writestr( - "agents_shipgate-0.16.0b1.dist-info/METADATA", - "Metadata-Version: 2.4\nName: agents-shipgate\nVersion: 0.16.0b1\n", + "agents_shipgate-0.16.0b2.dist-info/METADATA", + "Metadata-Version: 2.4\nName: agents-shipgate\nVersion: 0.16.0b2\n", ) @@ -124,9 +124,12 @@ def _production_result(wheel: Path) -> SafetyQualificationResultV1: matrices.extend( _confusion_matrix(cases, profile=profile) for profile in ( - "mcp", - "openapi", - "explicit_framework_inventory", + "mcp_openapi_declared_binding", + "openai_agents_sdk", + "langchain_crewai", + "google_adk", + "n8n", + "multi_agent_handoffs", "coding_agent_trust_roots", ) ) @@ -174,7 +177,7 @@ def _production_result(wheel: Path) -> SafetyQualificationResultV1: def _fixture(tmp_path: Path) -> tuple[Path, Path]: tmp_path.mkdir(parents=True, exist_ok=True) - wheel = tmp_path / "agents_shipgate-0.16.0b1-py3-none-any.whl" + wheel = tmp_path / "agents_shipgate-0.16.0b2-py3-none-any.whl" _write_wheel(wheel) qualification = tmp_path / "safety-qualification.json" qualification.write_text( @@ -225,7 +228,7 @@ def test_release_validator_rejects_wheel_hash_or_tag_mismatch(tmp_path: Path) -> verify_release_qualification( wheel_path=wheel, qualification_path=qualification, - tag="v0.16.0b2", + tag="v0.16.0b1", ) @@ -289,7 +292,7 @@ def test_release_validator_cli_fails_closed(tmp_path: Path) -> None: "--qualification", str(qualification), "--tag", - "v0.16.0b2", + "v0.16.0b1", ] ) == 1 diff --git a/tests/test_scan.py b/tests/test_scan.py index 5ae6f93f..5c22df44 100644 --- a/tests/test_scan.py +++ b/tests/test_scan.py @@ -305,6 +305,15 @@ def test_policy_evidence_source_threads_manifest_pointer(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: dangerous.write + source_id: tools + handoffs: [] + reason: reviewed test binding ci: mode: advisory """, @@ -382,6 +391,15 @@ def test_mixed_name_does_not_launder_keyword_effect_into_policy_blocker(tmp_path - id: api type: openapi path: openapi.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: get_or_delete_record + source_id: api + handoffs: [] + reason: reviewed test binding ci: mode: advisory """, @@ -454,6 +472,15 @@ def test_policy_evidence_source_resolves_existing_pointer_line(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: dangerous.write + source_id: tools + handoffs: [] + reason: reviewed test binding policies: require_approval_for_tools: - some.other.tool @@ -566,6 +593,22 @@ def test_change_me_placeholders_route_to_review_required(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: docs.lookup + source_id: tools + handoffs: [] + reason: reviewed test binding +action_surface: + actions: + - tool: docs.lookup + source_id: tools + effect: read + authority: + mode: none ci: mode: advisory """, @@ -584,7 +627,7 @@ def test_change_me_placeholders_route_to_review_required(tmp_path): assert any("shipgate.yaml:" in warning for warning in placeholder_warnings) assert report.release_decision is not None assert report.release_decision.evidence_coverage.source_warning_count >= 1 - assert report.release_decision.decision == "insufficient_evidence" + assert report.release_decision.decision == "review_required" # advisory mode does not fail CI, but the gate above is still routed. assert exit_code == 0 @@ -679,6 +722,22 @@ def test_severity_override_reranks_findings(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: docs.short + source_id: tools + handoffs: [] + reason: reviewed test binding +action_surface: + actions: + - tool: docs.short + source_id: tools + effect: read + authority: + mode: none checks: severity_overrides: SHIP-DOC-MISSING-DESCRIPTION: critical @@ -846,6 +905,15 @@ def write_manifest(effect: str) -> None: - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: process_order + source_id: tools + handoffs: [] + reason: reviewed test binding action_surface: actions: - tool: process_order @@ -938,6 +1006,15 @@ def test_baseline_scan_fails_only_on_new_findings(tmp_path): - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: support.lookup + source_id: tools + handoffs: [] + reason: reviewed test binding ci: mode: strict """, @@ -974,6 +1051,16 @@ def test_baseline_scan_fails_only_on_new_findings(tmp_path): """, encoding="utf-8", ) + config.write_text( + config.read_text(encoding="utf-8").replace( + " - tool: support.lookup\n source_id: tools\n", + " - tool: support.lookup\n" + " source_id: tools\n" + " - tool: billing.create_refund\n" + " source_id: tools\n", + ), + encoding="utf-8", + ) report, exit_code = run_scan( config_path=config, @@ -1093,6 +1180,17 @@ def test_same_name_tools_from_different_providers_remain_distinct(tmp_path): - id: mcp type: mcp path: mcp.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: shared.lookup + source_id: api + - tool: shared.lookup + source_id: mcp + handoffs: [] + reason: reviewed provider-specific test bindings """, encoding="utf-8", ) @@ -1161,6 +1259,15 @@ def test_manifest_scope_checks_read_only_purpose_with_write_tool(tmp_path): - id: api type: openapi path: openapi.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: ticket.create + source_id: api + handoffs: [] + reason: reviewed test binding permissions: scopes: - support:tickets:write @@ -1514,6 +1621,17 @@ def test_openapi_action_id_collision_degrades_instead_of_crashing(tmp_path): - id: goose-api type: openapi path: openapi.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: get_session + source_id: goose-api + - tool: get_session_detail + source_id: goose-api + handoffs: [] + reason: reviewed collision test bindings """, encoding="utf-8", ) @@ -1602,6 +1720,17 @@ def test_diff_from_report_with_duplicate_base_action_ids_degrades(tmp_path): - id: goose-api type: openapi path: openapi.yaml +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - tool: get_session + source_id: goose-api + - tool: get_session_detail + source_id: goose-api + handoffs: [] + reason: reviewed collision test bindings """, encoding="utf-8", ) diff --git a/tests/test_scenario_suggest.py b/tests/test_scenario_suggest.py index 396bb72d..c675c36a 100644 --- a/tests/test_scenario_suggest.py +++ b/tests/test_scenario_suggest.py @@ -121,18 +121,14 @@ def test_scenario_suggest_covers_reachable_active_scenario_findings(tmp_path): row_finding_ids = {row["finding_id"] for row in rows} assert reachable_active == row_finding_ids - wildcard = next( - finding - for finding in report["findings"] - if finding["check_id"] == "SHIP-INVENTORY-WILDCARD-TOOLS" - ) owner = next( finding for finding in report["findings"] if finding["check_id"] == "SHIP-MANIFEST-HIGH-RISK-OWNER-MISSING" ) - assert wildcard["id"] in row_finding_ids assert owner["id"] in row_finding_ids + assert any(row["name"] == "wildcard_mcp_tools.*" for row in report["tool_catalog"]) + assert all(row["name"] != "wildcard_mcp_tools.*" for row in report["tool_inventory"]) assert any( findings[finding_id]["severity"] == "medium" for finding_id in row_finding_ids diff --git a/tests/test_schema_boundaries.py b/tests/test_schema_boundaries.py index cfa35040..454477ab 100644 --- a/tests/test_schema_boundaries.py +++ b/tests/test_schema_boundaries.py @@ -274,7 +274,7 @@ def test_representative_schema_payloads_keep_wire_fields() -> None: tool_surface=ToolSurfaceSummary(total_tools=0, high_risk_tools=0), ) report_payload = report_json_payload(report) - assert report_payload["report_schema_version"] == "0.30" + assert report_payload["report_schema_version"] == "0.31" assert list(report_payload) == [ "schema_version", "report_schema_version", @@ -295,6 +295,8 @@ def test_representative_schema_payloads_keep_wire_fields() -> None: "tool_surface_diff", "action_surface_facts", "action_surface_diff", + "binding_surface_facts", + "binding_surface_diff", "capability_runtime_evidence", "api_surface", "anthropic_surface", @@ -308,6 +310,7 @@ def test_representative_schema_payloads_keep_wire_fields() -> None: "loaded_plugins", "loaded_adapters", "tool_inventory", + "tool_catalog", "source_warnings", "agent_summary", "policy_audit", @@ -352,7 +355,7 @@ def test_representative_schema_payloads_keep_wire_fields() -> None: not_proven=NotProvenSection(headline="not proven"), ) packet_payload = serialize_packet_json(packet) - assert packet_payload["packet_schema_version"] == "0.9" + assert packet_payload["packet_schema_version"] == "0.10" assert "generated_at" not in packet_payload assert "action_surface_diff" in packet_payload assert report_payload["capability_runtime_evidence"]["enabled"] is False diff --git a/tests/test_semantic_cold_start.py b/tests/test_semantic_cold_start.py index 0e1eedb4..4ec2262d 100644 --- a/tests/test_semantic_cold_start.py +++ b/tests/test_semantic_cold_start.py @@ -28,12 +28,13 @@ def test_unaugmented_framework_cold_start_is_measured_as_insufficient_evidence( assert exit_code == 0 assert report.release_decision.decision == "insufficient_evidence" - coverage = report.release_decision.evidence_coverage.semantic_coverage - assert coverage.total_actions == 1 - assert coverage.pass_eligible_actions == 0 + assert len(report.tool_catalog) == 1 + assert report.tool_inventory == [] + coverage = report.release_decision.evidence_coverage.binding_coverage + assert coverage.reachable_tools == 0 + assert coverage.unbound_tools == 1 assert coverage.gap_count >= 1 gap_kinds = { gap.kind for gap in report.release_decision.evidence_coverage.evidence_gaps } - assert "incomplete_surface" in gap_kinds - assert "missing_authority_evidence" in gap_kinds + assert "ambiguous_root_agent" in gap_kinds diff --git a/tests/test_tool_identity.py b/tests/test_tool_identity.py index b383daff..bc23b509 100644 --- a/tests/test_tool_identity.py +++ b/tests/test_tool_identity.py @@ -278,6 +278,14 @@ def test_stale_suppression_routes_to_catalog_review_not_identity_gap(tmp_path) - - id: support type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {tool: lookup_case, source_id: support} + handoffs: [] + reason: reviewed stale-suppression fixture binding checks: ignore: - check_id: SHIP-DOC-MISSING-DESCRIPTION diff --git a/tests/test_toolkit_bounds_check.py b/tests/test_toolkit_bounds_check.py index 42845b27..b7d5388e 100644 --- a/tests/test_toolkit_bounds_check.py +++ b/tests/test_toolkit_bounds_check.py @@ -106,7 +106,7 @@ def test_scan_of_stripe_head_fixture_names_the_unbounded_toolkit(tmp_path: Path) # The finding must point at the constructor, not shipgate.yaml. source = matches[0].get("source") or {} assert (source.get("path") or "").endswith("support_agent.py"), source - # Phase 2c: the named high concern elevates the verdict out of the vague - # insufficient_evidence into an actionable review_required (the pilot's - # silent/IE case is now a routed human review). - assert report["release_decision"]["decision"] == "review_required" + # The toolkit remains only possibly reachable because its binding graph is + # incomplete. The named finding stays visible, while the verdict remains + # fail-closed on missing binding evidence. + assert report["release_decision"]["decision"] == "insufficient_evidence" diff --git a/tests/test_v07_metadata_roundtrip.py b/tests/test_v07_metadata_roundtrip.py index 1c3c57e8..167d4425 100644 --- a/tests/test_v07_metadata_roundtrip.py +++ b/tests/test_v07_metadata_roundtrip.py @@ -240,7 +240,7 @@ def test_package_version_is_current_in_tree_runtime(): """Guard against bumping schemas while leaving package metadata behind.""" import agents_shipgate - assert agents_shipgate.__version__ == "0.16.0b1", ( + assert agents_shipgate.__version__ == "0.16.0b2", ( f"package version is {agents_shipgate.__version__!r}; " - "expected 0.16.0b1 for the current in-tree runtime" + "expected 0.16.0b2 for the current in-tree runtime" ) diff --git a/tests/test_validation_evidence.py b/tests/test_validation_evidence.py index 93b46065..6932c243 100644 --- a/tests/test_validation_evidence.py +++ b/tests/test_validation_evidence.py @@ -649,6 +649,14 @@ def _write_project( - id: tools type: mcp path: tools.json +agent_bindings: + declarations: + - agent: root + complete: true + tools: + - {{tool: issue_refund, source_id: tools}} + handoffs: [] + reason: reviewed validation fixture binding {validation_block} permissions: scopes: diff --git a/tests/test_verifier_scenarios.py b/tests/test_verifier_scenarios.py index 8b03dad7..63101b0f 100644 --- a/tests/test_verifier_scenarios.py +++ b/tests/test_verifier_scenarios.py @@ -19,6 +19,7 @@ import subprocess from pathlib import Path +import yaml from typer.testing import CliRunner from agents_shipgate.cli.main import app @@ -50,6 +51,7 @@ "name": "support.search_kb", "description": "Search the support knowledge base.", "annotations": {"readOnlyHint": True, "idempotentHint": True}, + "auth": {"mode": "none"}, "inputSchema": { "type": "object", "properties": {"q": {"type": "string"}}, @@ -115,6 +117,27 @@ def _export_capability_lock(repo: Path) -> None: def _write_tools(repo: Path, payload: dict) -> None: (repo / "tools.json").write_text(json.dumps(payload, indent=2), encoding="utf-8") + manifest_path = repo / "shipgate.yaml" + if manifest_path.is_file(): + manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + manifest["agent_bindings"] = { + "declarations": [ + { + "agent": "root", + "complete": True, + "tools": [ + {"tool": tool["name"], "source_id": "tools"} + for tool in payload["tools"] + ], + "handoffs": [], + "reason": "reviewed verifier scenario binding", + } + ] + } + manifest_path.write_text( + yaml.safe_dump(manifest, sort_keys=False), + encoding="utf-8", + ) def _verify(repo: Path) -> dict: @@ -154,7 +177,7 @@ def test_scenario_codex_adds_refund_tool_blocks(tmp_path: Path) -> None: assert payload["head_status"] == "succeeded" assert payload["merge_verdict"] == "blocked" assert payload["can_merge_without_human"] is False - assert payload["capability_review"]["trust_root_touched"] is False + assert payload["capability_review"]["trust_root_touched"] is True # The top capability change references the money-moving refund action. refund_adds = [ c @@ -223,7 +246,7 @@ def test_scenario_committed_base_lock_emits_semantic_capability_diff( ) reports = repo / "agents-shipgate-reports" diff_payload = json.loads((reports / "capability-lock-diff.json").read_text(encoding="utf-8")) - assert diff_payload["capability_lock_diff_schema_version"] == "0.5" + assert diff_payload["capability_lock_diff_schema_version"] == "0.6" assert diff_payload["summary"]["added"] == 1 assert diff_payload["summary"]["changed"] == 0 assert diff_payload["added"][0]["identity"]["tool_name"] == "stripe.create_refund" diff --git a/tests/test_verify.py b/tests/test_verify.py index 03d1e76e..8a93f85e 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -463,7 +463,9 @@ def test_verify_real_base_scan_enables_head_diff(tmp_path: Path) -> None: (repo / "agents-shipgate-reports" / "report.json").read_text(encoding="utf-8") ) assert report_payload["tool_surface_diff"]["enabled"] is True - assert report_payload["tool_surface_diff"]["summary"]["tools_added"] == 1 + assert report_payload["tool_surface_diff"]["summary"]["tools_added"] == 0 + assert len(report_payload["tool_catalog"]) == 1 + assert report_payload["release_decision"]["decision"] == "insufficient_evidence" def test_pr_comment_keeps_code_span_values_unescaped() -> None: diff --git a/tests/test_verify_config_binding.py b/tests/test_verify_config_binding.py index 9c74f4c6..764ab820 100644 --- a/tests/test_verify_config_binding.py +++ b/tests/test_verify_config_binding.py @@ -485,9 +485,9 @@ def test_config_binding_removal_routes_to_review(tmp_path): verification_context=VerificationContext(changed_files=["support_agent.py"]), packet_enabled=False, ) - # The statically-visible binding removal lifts the verdict out of silent - # insufficient_evidence parity. - assert head_report.release_decision.decision in {"blocked", "review_required"} + # The statically-visible removal remains fail-closed: without a complete + # root-reachable graph, the named concern cannot become a pass. + assert head_report.release_decision.decision == "insufficient_evidence" gating = { item.check_id for item in [ From 8e0e1e46734c6eebb8a379a826c0bda234b9b15d Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Sat, 11 Jul 2026 21:47:11 -0700 Subject: [PATCH 2/3] Harden binding evidence trust boundary --- .github/workflows/ci.yml | 2 +- src/agents_shipgate/ci/release_decision.py | 26 +++++ src/agents_shipgate/cli/scan/tools_agent.py | 1 + src/agents_shipgate/core/agent_bindings.py | 91 +++++++++++++++++- src/agents_shipgate/core/domain.py | 20 ++++ .../core/semantic_consistency.py | 1 + src/agents_shipgate/inputs/common.py | 26 +++++ src/agents_shipgate/inputs/mcp.py | 15 ++- .../inputs/openai_sdk_static.py | 95 +++++++++---------- src/agents_shipgate/inputs/openapi.py | 27 +++++- tests/test_agent_bindings.py | 84 +++++++++++++++- tests/test_inputs.py | 44 +++++++++ tests/test_p0_binding_canaries.py | 77 ++++++++++++++- 13 files changed, 450 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73531c32..9d2f812f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: # verdict regression should fail with a named canary before coverage, # integration, or performance noise can obscure the safety signal. - - name: Agent binding safety canaries (48/48 exact outcomes) + - name: Agent binding safety canaries run: python -m pytest tests/test_p0_binding_canaries.py -q # Binding is a separate trust dimension from effect and authority. # Keep these exact outcomes visible before the aggregate coverage run. diff --git a/src/agents_shipgate/ci/release_decision.py b/src/agents_shipgate/ci/release_decision.py index 9c1d1759..747f8058 100644 --- a/src/agents_shipgate/ci/release_decision.py +++ b/src/agents_shipgate/ci/release_decision.py @@ -431,6 +431,32 @@ def _binding_coverage( ), ) ) + covered_possible_ids = { + issue.tool_id + for issue in graph.issues + if issue.kind == "partial_binding_evidence" and issue.tool_id is not None + } + for tool_id in sorted(set(graph.possible_tool_ids) - covered_possible_ids): + _increment(reason_counts, "partial_binding_evidence") + gaps.append( + EvidenceGap( + kind="partial_binding_evidence", + subject=tool_id, + why="A possibly reachable tool lacks a complete static binding edge.", + next_action=EvidenceGapAction( + kind="provide_complete_binding_graph", + command=_SEMANTIC_RERUN_COMMAND, + path="shipgate.yaml#agent_bindings", + why="Possibly reachable tools cannot be omitted from the release decision.", + expects="Provide static wiring or an exact reviewed declaration.", + accepted_values=[ + "literal_tools", + "literal_handoffs", + "complete:true", + ], + ), + ) + ) return ( BindingCoverageDecision( total_catalog_tools=( diff --git a/src/agents_shipgate/cli/scan/tools_agent.py b/src/agents_shipgate/cli/scan/tools_agent.py index 056bb507..6f7ff102 100644 --- a/src/agents_shipgate/cli/scan/tools_agent.py +++ b/src/agents_shipgate/cli/scan/tools_agent.py @@ -48,6 +48,7 @@ def _build_tools_and_agent( manifest, tool_catalog, inputs.artifact_bag, + inputs.loaded_sources, ) reachable_ids = set(binding_graph.reachable_tool_ids) declarations, tool_catalog = resolve_selectors_by_tool_id( diff --git a/src/agents_shipgate/core/agent_bindings.py b/src/agents_shipgate/core/agent_bindings.py index aa009f1a..17b034d6 100644 --- a/src/agents_shipgate/core/agent_bindings.py +++ b/src/agents_shipgate/core/agent_bindings.py @@ -15,6 +15,7 @@ from agents_shipgate.core.artifacts import ArtifactBag from agents_shipgate.core.domain import ( BindingSemanticAssessment, + LoadedToolSource, SemanticClaim, SemanticIssue, Tool, @@ -86,6 +87,7 @@ def resolve_agent_binding_graph( manifest: AgentsShipgateManifest, tools: list[Tool], artifacts: ArtifactBag, + loaded_sources: list[LoadedToolSource] | None = None, ) -> tuple[AgentBindingGraphAssessment, list[Tool]]: """Resolve the exact static graph rooted at the configured agent. @@ -99,7 +101,7 @@ def resolve_agent_binding_graph( raw_handoffs, partials, invalid_annotations, - ) = _observations(tools, artifacts) + ) = _observations(tools, artifacts, loaded_sources or []) declarations = manifest.agent_bindings.declarations for declaration in declarations: if declaration.agent != "root": @@ -368,6 +370,20 @@ def resolve_agent_binding_graph( tool_edges = _dedupe_tool_edges(tool_edges) handoff_edges = _dedupe_handoff_edges(handoff_edges) reachable_agents, paths = _reachable_agents(root, handoff_edges) + for edge in handoff_edges: + if edge.source_agent_id in reachable_agents and not edge.complete: + issues.append( + AgentBindingIssue( + kind="incomplete_handoff_graph", + message=( + "A handoff from a reachable agent is incomplete; its downstream " + "capability surface cannot be proven." + ), + agent_id=edge.source_agent_id, + source=edge.source, + source_pointer=edge.source_pointer, + ) + ) reachable_ids = sorted( { edge.tool_id @@ -385,6 +401,30 @@ def resolve_agent_binding_graph( ) unbound_ids = sorted(set(selector_index.by_id) - set(reachable_ids) - set(possible_ids)) + # An incomplete positive edge means the tool may be reachable. It is not + # part of the proven capability surface, but it must still fail closed as + # a first-class evidence gap rather than disappearing from assessment. + incomplete_by_tool = { + edge.tool_id: edge + for edge in tool_edges + if edge.agent_id in reachable_agents and not edge.complete + } + for tool_id in possible_ids: + edge = incomplete_by_tool[tool_id] + issues.append( + AgentBindingIssue( + kind="partial_binding_evidence", + message=( + "A possibly reachable tool has an incomplete binding edge; " + "provide a complete static graph or reviewed declaration." + ), + agent_id=edge.agent_id, + tool_id=tool_id, + source=edge.source, + source_pointer=edge.source_pointer, + ) + ) + if root is not None and not declarations and not tool_edges and tools: issues.append( AgentBindingIssue( @@ -457,7 +497,7 @@ def resolve_agent_binding_graph( def _observations( - tools: list[Tool], artifacts: ArtifactBag + tools: list[Tool], artifacts: ArtifactBag, loaded_sources: list[LoadedToolSource] ) -> tuple[ list[_AgentObservation], list[_RawToolEdge], @@ -471,6 +511,53 @@ def _observations( partials: set[str] = set() invalid_annotations: set[str] = set() + for loaded in loaded_sources: + for observation in loaded.binding_observations: + complete = observation.tools_complete and observation.handoffs_complete + agents.append( + _AgentObservation( + observation.agent, + observation.source_id, + observation.source, + observation.source_pointer, + complete, + ) + ) + for tool_name in observation.tool_names: + edges.append( + _RawToolEdge( + observation.agent, + observation.source_id, + tool_name, + "direct_tool", + observation.source, + observation.source_pointer, + observation.tools_complete, + ) + ) + for target_agent in observation.handoff_names: + agents.append( + _AgentObservation( + target_agent, + observation.source_id, + observation.source, + observation.source_pointer, + observation.handoffs_complete, + ) + ) + handoffs.append( + _RawHandoffEdge( + observation.agent, + target_agent, + observation.source_id, + "handoff", + observation.source, + observation.source_pointer, + observation.handoffs_complete, + ) + ) + partials.update(observation.issues) + for tool in tools: for reason in _string_list(tool.annotations.get("binding_surface_partial")): partials.add(reason) diff --git a/src/agents_shipgate/core/domain.py b/src/agents_shipgate/core/domain.py index 2c9017e7..2e6bf79e 100644 --- a/src/agents_shipgate/core/domain.py +++ b/src/agents_shipgate/core/domain.py @@ -346,6 +346,22 @@ class ToolkitScopeBound(BaseModel): config_path: str | None = None +class AgentBindingObservation(BaseModel): + """One framework parser's normalized, agent-level binding observation.""" + + model_config = ConfigDict(extra="forbid") + + agent: str + source_id: str | None = None + source: str + source_pointer: str | None = None + tool_names: list[str] = Field(default_factory=list) + handoff_names: list[str] = Field(default_factory=list) + tools_complete: bool = True + handoffs_complete: bool = True + issues: list[str] = Field(default_factory=list) + + class LoadedToolSource(BaseModel): source_id: str source_type: str @@ -355,6 +371,10 @@ class LoadedToolSource(BaseModel): # toolkits found in this source (e.g. ``stripe_agent_toolkit``). Empty # for sources that declare no recognized agent-toolkit constructor. toolkit_bounds: list[ToolkitScopeBound] = Field(default_factory=list) + # Framework-owned observations about Agent(...) wiring. These are kept + # separate from Tool.annotations so catalog-controlled metadata can never + # become authority-bearing binding evidence. + binding_observations: list[AgentBindingObservation] = Field(default_factory=list) # --------------------------------------------------------------------------- diff --git a/src/agents_shipgate/core/semantic_consistency.py b/src/agents_shipgate/core/semantic_consistency.py index fbfbf938..13113291 100644 --- a/src/agents_shipgate/core/semantic_consistency.py +++ b/src/agents_shipgate/core/semantic_consistency.py @@ -120,6 +120,7 @@ def validate_semantic_consistency( coverage.gap_count or binding_coverage.gap_count or not graph.pass_eligible + or bool(graph.possible_tool_ids) or coverage.review_concern_count or identity_coverage.gap_count or identity_eligible != len(assessments) diff --git a/src/agents_shipgate/inputs/common.py b/src/agents_shipgate/inputs/common.py index 7ccf9a0c..7b6ec1c7 100644 --- a/src/agents_shipgate/inputs/common.py +++ b/src/agents_shipgate/inputs/common.py @@ -14,6 +14,32 @@ from agents_shipgate.core.domain import ToolParameter from agents_shipgate.core.errors import InputParseError +# These keys are authority-bearing inputs to the binding resolver. Catalog +# formats may carry arbitrary extension metadata, so their loaders must never +# pass these names through from source-controlled annotation dictionaries. +RESERVED_BINDING_ANNOTATION_KEYS = frozenset( + { + "agent_bindings", + "agent_handoffs", + "adk_agent_name", + "adk_agent_source_id", + "binding_surface_partial", + } +) + + +def strip_untrusted_binding_annotations( + annotations: dict[str, Any], +) -> tuple[dict[str, Any], list[str]]: + """Remove binding-resolver keys from an untrusted catalog annotation map.""" + + rejected = sorted(RESERVED_BINDING_ANNOTATION_KEYS.intersection(annotations)) + return ( + {key: value for key, value in annotations.items() if key not in rejected}, + rejected, + ) + + HTTP_METHODS = {"get", "put", "post", "delete", "patch", "options", "head", "trace"} MAX_INPUT_FILE_BYTES = 10 * 1024 * 1024 CONVENTIONAL_TOOL_NAME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9._-]{0,128}$") diff --git a/src/agents_shipgate/inputs/mcp.py b/src/agents_shipgate/inputs/mcp.py index fce72ff8..627fd48e 100644 --- a/src/agents_shipgate/inputs/mcp.py +++ b/src/agents_shipgate/inputs/mcp.py @@ -11,6 +11,7 @@ resolve_input_path, schema_to_parameters, stable_tool_id, + strip_untrusted_binding_annotations, tool_name_warning, ) from agents_shipgate.inputs.protocol import LoadedAdapterResult @@ -97,7 +98,17 @@ def load_mcp_tools(source: ToolSourceConfig, base_dir: Path) -> LoadedToolSource warnings.append(warning) input_schema = _first_present(raw, ["inputSchema", "input_schema"]) or {} output_schema = _first_present(raw, ["outputSchema", "output_schema"]) or {} - annotations = raw.get("annotations") or {} + raw_annotations = raw.get("annotations") or {} + annotations: dict[str, Any] = {} + if isinstance(raw_annotations, dict): + annotations, rejected_binding_keys = strip_untrusted_binding_annotations( + raw_annotations + ) + if rejected_binding_keys: + warnings.append( + f"MCP tool {name_text!r} contains reserved binding annotations " + f"that were ignored: {', '.join(rejected_binding_keys)}" + ) raw_auth = raw.get("auth") auth_explicit = "auth" in raw pointer = f"{pointer_prefix}/{index}" @@ -123,7 +134,7 @@ def load_mcp_tools(source: ToolSourceConfig, base_dir: Path) -> LoadedToolSource input_schema=input_schema if isinstance(input_schema, dict) else {}, output_schema=output_schema if isinstance(output_schema, dict) else {}, parameters=schema_to_parameters(input_schema), - annotations=annotations if isinstance(annotations, dict) else {}, + annotations=annotations, auth=_mcp_auth_info(raw_auth, explicit=auth_explicit), owner=raw.get("owner"), extraction_confidence="high", diff --git a/src/agents_shipgate/inputs/openai_sdk_static.py b/src/agents_shipgate/inputs/openai_sdk_static.py index ad8aad1f..d8fffbe2 100644 --- a/src/agents_shipgate/inputs/openai_sdk_static.py +++ b/src/agents_shipgate/inputs/openai_sdk_static.py @@ -5,6 +5,7 @@ from typing import ClassVar, Literal from agents_shipgate.core.domain import ( + AgentBindingObservation, AuthInfo, LoadedToolSource, Tool, @@ -66,12 +67,15 @@ def load_openai_sdk_static_tools( raise InputParseError( f"OpenAI Agents SDK source must be a Python file or directory: {path}" ) - binding_warnings = _attach_agent_bindings(tools, python_files, source, base_dir) + binding_warnings, binding_observations = _extract_agent_bindings( + tools, python_files, source, base_dir + ) return LoadedToolSource( source_id=source.id, source_type="openai_agents_sdk", tools=tools, toolkit_bounds=toolkit_bounds, + binding_observations=binding_observations, warnings=[*_toolkit_binding_warnings(toolkit_bounds), *binding_warnings], ) @@ -124,13 +128,13 @@ def _load_python_file( return tools, _detect_toolkit_bounds(tree, ref) -def _attach_agent_bindings( +def _extract_agent_bindings( tools: list[Tool], paths: list[Path], source: ToolSourceConfig, base_dir: Path, -) -> list[str]: - """Attach exact, local-only ``Agent(..., tools=[...])`` wiring. +) -> tuple[list[str], list[AgentBindingObservation]]: + """Extract exact, local-only ``Agent(..., tools=[...])`` wiring. This intentionally resolves only literal lists, names bound to literal lists, and local/imported function-tool identifiers. Dynamic expressions @@ -138,6 +142,7 @@ def _attach_agent_bindings( """ warnings: list[str] = [] + observations: list[AgentBindingObservation] = [] tool_by_name = {tool.name: tool for tool in tools} tool_by_name.update( { @@ -170,63 +175,55 @@ def _attach_agent_bindings( tools_expr = _keyword(call, "tools") names = _resolve_name_list(tools_expr, list_vars, import_aliases) pointer = f"{source_ref}:{call.lineno}" + issues: list[str] = [] + tools_complete = True if names is None: reason = ( f"OpenAI Agents SDK agent {target!r} at {pointer} uses a " "dynamic tools expression; its binding graph is incomplete." ) warnings.append(reason) - for tool in tools: - if tool.source_id == source.id: - tool.annotations.setdefault("binding_surface_partial", []).append(reason) - continue - for name in names: - tool = tool_by_name.get(name) - if tool is None: - reason = ( - f"OpenAI Agents SDK agent {target!r} at {pointer} binds " - f"unresolved tool {name!r}." - ) - warnings.append(reason) - for candidate in tools: - if candidate.source_id == source.id: - candidate.annotations.setdefault("binding_surface_partial", []).append(reason) - continue - tool.annotations.setdefault("agent_bindings", []).append( - { - "agent": target, - "source_id": source.id, - "edge_type": "direct_tool", - "source": source_ref, - "source_pointer": pointer, - "complete": True, - } - ) + issues.append(reason) + tools_complete = False + names = [] + else: + for name in names: + if tool_by_name.get(name) is None: + reason = ( + f"OpenAI Agents SDK agent {target!r} at {pointer} binds " + f"unresolved tool {name!r}." + ) + warnings.append(reason) + issues.append(reason) + tools_complete = False + names = [ + tool_by_name[name].name if name in tool_by_name else name + for name in names + ] handoff_names = _resolve_name_list( _keyword(call, "handoffs"), list_vars, import_aliases ) + handoffs_complete = True if handoff_names is None: reason = f"OpenAI Agents SDK agent {target!r} has dynamic handoffs at {pointer}." warnings.append(reason) - for tool in tools: - if tool.source_id == source.id: - tool.annotations.setdefault("binding_surface_partial", []).append(reason) - elif handoff_names: - anchor = next((tool_by_name.get(name) for name in names if tool_by_name.get(name)), None) - if anchor is not None: - anchor.annotations.setdefault("agent_handoffs", []).extend( - { - "source_agent": target, - "target_agent": handoff, - "source_id": source.id, - "edge_type": "handoff", - "source": source_ref, - "source_pointer": pointer, - "complete": True, - } - for handoff in handoff_names - ) - return list(dict.fromkeys(warnings)) + issues.append(reason) + handoffs_complete = False + handoff_names = [] + observations.append( + AgentBindingObservation( + agent=target, + source_id=source.id, + source=source_ref, + source_pointer=pointer, + tool_names=names, + handoff_names=handoff_names, + tools_complete=tools_complete, + handoffs_complete=handoffs_complete, + issues=issues, + ) + ) + return list(dict.fromkeys(warnings)), observations def _assignment_target(node: ast.Assign | ast.AnnAssign) -> str | None: diff --git a/src/agents_shipgate/inputs/openapi.py b/src/agents_shipgate/inputs/openapi.py index 6c54ca32..a65b6185 100644 --- a/src/agents_shipgate/inputs/openapi.py +++ b/src/agents_shipgate/inputs/openapi.py @@ -22,6 +22,7 @@ resolve_input_path, schema_to_parameters, stable_tool_id, + strip_untrusted_binding_annotations, tool_name_warning, ) from agents_shipgate.inputs.protocol import LoadedAdapterResult @@ -73,6 +74,7 @@ def load_openapi_tools(source: ToolSourceConfig, base_dir: Path) -> LoadedToolSo operation=operation, path_parameters=path_parameters, positions=positions, + warnings=warnings, ) except (RecursionError, MemoryError): raise @@ -114,6 +116,7 @@ def _operation_to_tool( operation: dict[str, Any], path_parameters: list[Any], positions: PositionIndex, + warnings: list[str], ) -> Tool: operation_id = operation.get("operationId") or _operation_name(method, api_path) request_schema = _extract_request_schema(document, operation) @@ -124,7 +127,11 @@ def _operation_to_tool( description = "\n".join( part for part in [operation.get("summary"), operation.get("description")] if part ) - annotations = _extract_annotations(operation) + annotations = _extract_annotations( + operation, + operation_label=str(operation_id), + warnings=warnings, + ) annotations["httpMethod"] = method.upper() annotations["path"] = api_path auth = _extract_security(document, operation) @@ -330,7 +337,12 @@ def _extract_security(document: dict[str, Any], operation: dict[str, Any]) -> Au ) -def _extract_annotations(operation: dict[str, Any]) -> dict[str, Any]: +def _extract_annotations( + operation: dict[str, Any], + *, + operation_label: str, + warnings: list[str], +) -> dict[str, Any]: annotations: dict[str, Any] = {} for source_key, output_key in { "x-readOnlyHint": "readOnlyHint", @@ -343,7 +355,16 @@ def _extract_annotations(operation: dict[str, Any]) -> dict[str, Any]: annotations[output_key] = operation[source_key] agents_shipgate = operation.get("x-agents-shipgate") if isinstance(agents_shipgate, dict): - annotations.update(agents_shipgate) + safe_annotations, rejected_binding_keys = strip_untrusted_binding_annotations( + agents_shipgate + ) + annotations.update(safe_annotations) + if rejected_binding_keys: + warnings.append( + f"OpenAPI operation {operation_label!r} contains reserved binding " + "annotations that were ignored: " + f"{', '.join(rejected_binding_keys)}" + ) return annotations diff --git a/tests/test_agent_bindings.py b/tests/test_agent_bindings.py index 37933e6d..df956a5a 100644 --- a/tests/test_agent_bindings.py +++ b/tests/test_agent_bindings.py @@ -2,7 +2,12 @@ from agents_shipgate.core.agent_bindings import resolve_agent_binding_graph from agents_shipgate.core.artifacts import ArtifactBag -from agents_shipgate.core.domain import AuthInfo, Tool +from agents_shipgate.core.domain import ( + AgentBindingObservation, + AuthInfo, + LoadedToolSource, + Tool, +) from agents_shipgate.schemas.manifest import AgentsShipgateManifest @@ -163,3 +168,80 @@ def test_dynamic_binding_annotation_fails_closed() -> None: assert graph.status == "partial" assert graph.pass_eligible is False assert "partial_binding_evidence" in {issue.kind for issue in graph.issues} + + +def test_incomplete_positive_edge_creates_tool_specific_gap() -> None: + tool = _tool("exfiltrate", agent="root_agent") + tool.annotations["agent_bindings"][0]["complete"] = False + + graph, _ = resolve_agent_binding_graph( + _manifest(sdk_object="root_agent"), [tool], ArtifactBag() + ) + + assert graph.reachable_tool_ids == [] + assert graph.possible_tool_ids == [tool.id] + assert graph.status == "partial" + assert graph.pass_eligible is False + assert any( + issue.kind == "partial_binding_evidence" and issue.tool_id == tool.id + for issue in graph.issues + ) + + +def test_agent_level_observation_preserves_toolless_router_handoff() -> None: + worker_tool = _tool("lookup", source_id="sdk") + loaded = LoadedToolSource( + source_id="sdk", + source_type="openai_agents_sdk", + tools=[worker_tool], + binding_observations=[ + AgentBindingObservation( + agent="router", + source_id="sdk", + source="agent.py", + source_pointer="agent.py:4", + handoff_names=["worker"], + ), + AgentBindingObservation( + agent="worker", + source_id="sdk", + source="agent.py", + source_pointer="agent.py:5", + tool_names=["lookup"], + ), + ], + ) + + graph, _ = resolve_agent_binding_graph( + _manifest(sdk_object="router"), + [worker_tool], + ArtifactBag(), + [loaded], + ) + + assert graph.pass_eligible is True + assert graph.reachable_tool_ids == [worker_tool.id] + assert len(graph.handoff_edges) == 1 + + +def test_incomplete_handoff_from_reachable_agent_fails_closed() -> None: + root_tool = _tool("route", source_id="sdk", agent="root_agent") + root_tool.annotations["agent_handoffs"] = [ + { + "source_agent": "root_agent", + "target_agent": "worker", + "source_id": "sdk", + "complete": False, + } + ] + worker_tool = _tool("lookup", source_id="sdk", agent="worker") + + graph, _ = resolve_agent_binding_graph( + _manifest(sdk_object="root_agent"), + [root_tool, worker_tool], + ArtifactBag(), + ) + + assert graph.reachable_tool_ids == [root_tool.id] + assert graph.pass_eligible is False + assert "incomplete_handoff_graph" in {issue.kind for issue in graph.issues} diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 461b539f..e5cc1c90 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -68,6 +68,21 @@ def test_mcp_loader_accepts_array_root(tmp_path): assert [tool.name for tool in loaded.tools] == ["support.lookup"] +def test_mcp_loader_strips_reserved_binding_annotations(tmp_path): + (tmp_path / "tools.json").write_text( + '{"tools":[{"name":"danger","annotations":{' + '"readOnlyHint":true,"agent_bindings":[{"agent":"root"}],' + '"agent_handoffs":[],"adk_agent_name":"root",' + '"adk_agent_source_id":"catalog","binding_surface_partial":[]}}]}', + encoding="utf-8", + ) + + loaded = load_mcp_tools(ToolSourceConfig(id="catalog", type="mcp", path="tools.json"), tmp_path) + + assert loaded.tools[0].annotations == {"readOnlyHint": True} + assert any("reserved binding annotations" in warning for warning in loaded.warnings) + + def test_mcp_wildcard_with_tools_is_rejected(tmp_path): tools_path = tmp_path / "tools.json" tools_path.write_text( @@ -449,6 +464,35 @@ def summarize(case_id: str) -> str: assert [tool.name for tool in loaded.tools] == ["support.lookup", "summarize"] +def test_openai_sdk_static_records_toolless_router_handoffs(tmp_path): + agent_path = tmp_path / "agent.py" + agent_path.write_text( + """ +from agents import Agent, function_tool + +@function_tool +def lookup(query: str) -> str: + return "" + +worker = Agent(tools=[lookup]) +router = Agent(tools=[], handoffs=[worker]) +""", + encoding="utf-8", + ) + + manifest = load_manifest(BASE / "shipgate.yaml") + loaded = load_openai_sdk_static_tools( + ToolSourceConfig(id="sdk", type="openai_agents_sdk", path="agent.py"), + manifest, + tmp_path, + ) + + by_agent = {observation.agent: observation for observation in loaded.binding_observations} + assert by_agent["router"].tool_names == [] + assert by_agent["router"].handoff_names == ["worker"] + assert by_agent["router"].handoffs_complete is True + + def test_openai_sdk_static_directory_scans_sorted_python_files(tmp_path): agents_dir = tmp_path / "agents" agents_dir.mkdir() diff --git a/tests/test_p0_binding_canaries.py b/tests/test_p0_binding_canaries.py index 41a8e7a6..bbba980e 100644 --- a/tests/test_p0_binding_canaries.py +++ b/tests/test_p0_binding_canaries.py @@ -2,10 +2,12 @@ import pytest +from agents_shipgate.cli.scan import run_scan from agents_shipgate.core.agent_bindings import resolve_agent_binding_graph from agents_shipgate.core.artifacts import ArtifactBag from agents_shipgate.core.domain import Tool -from agents_shipgate.schemas.manifest import AgentsShipgateManifest +from agents_shipgate.inputs.openapi import load_openapi_tools +from agents_shipgate.schemas.manifest import AgentsShipgateManifest, ToolSourceConfig def _manifest(binding_mode: str) -> AgentsShipgateManifest: @@ -116,3 +118,76 @@ def test_48_binding_canaries_have_exact_outcomes( assert graph.status == status assert graph.pass_eligible is pass_eligible assert len(graph.reachable_tool_ids) == reachable_count + + +def test_mcp_catalog_cannot_self_declare_agent_bindings(tmp_path) -> None: + (tmp_path / "tools.json").write_text( + """{"tools":[ + {"name":"read_docs","annotations":{"readOnlyHint":true, + "agent_bindings":[{"agent":"root","complete":true}]}}, + {"name":"exfiltrate_and_wire_funds","annotations":{"readOnlyHint":true, + "agent_bindings":[{"agent":"root","complete":false}]}} + ]}""", + encoding="utf-8", + ) + (tmp_path / "shipgate.yaml").write_text( + """{ + "version":"0.1", + "project":{"name":"hostile-catalog"}, + "agent":{"name":"agent","declared_purpose":["test trust boundary"]}, + "environment":{"target":"local"}, + "tool_sources":[{"id":"source","type":"mcp","path":"tools.json"}] + }""", + encoding="utf-8", + ) + report, exit_code = run_scan( + config_path=tmp_path / "shipgate.yaml", + output_dir=tmp_path / "reports", + formats=["json"], + ci_mode="strict", + packet_enabled=False, + ) + + graph = report.binding_surface_facts + assert report.release_decision is not None + assert report.release_decision.decision == "insufficient_evidence" + assert report.release_decision.fail_policy.would_fail_ci is True + assert exit_code == 20 + assert graph.tool_edges == [] + assert graph.reachable_tool_ids == [] + assert graph.possible_tool_ids == [] + assert graph.pass_eligible is False + assert report.tool_inventory == [] + assert {entry["name"] for entry in report.tool_catalog} == { + "read_docs", + "exfiltrate_and_wire_funds", + } + + +def test_openapi_catalog_cannot_self_declare_agent_bindings(tmp_path) -> None: + (tmp_path / "openapi.json").write_text( + """{ + "openapi":"3.1.0", + "info":{"title":"hostile","version":"1"}, + "paths":{"/wire":{"post":{"operationId":"wire_funds", + "security":[], + "x-agents-shipgate":{"agent_bindings":[ + {"agent":"root","complete":true} + ]}, + "responses":{"200":{"description":"ok"}} + }}} + }""", + encoding="utf-8", + ) + loaded = load_openapi_tools( + ToolSourceConfig(id="source", type="openapi", path="openapi.json"), tmp_path + ) + + graph, _ = resolve_agent_binding_graph( + _manifest("catalog"), loaded.tools, ArtifactBag(), [loaded] + ) + + assert graph.tool_edges == [] + assert graph.reachable_tool_ids == [] + assert graph.pass_eligible is False + assert any("reserved binding annotations" in warning for warning in loaded.warnings) From a0a950fec557028dbd2df2a02b90f34bf27701f9 Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Sat, 11 Jul 2026 22:53:54 -0700 Subject: [PATCH 3/3] Sanitize codex MCP binding annotations --- src/agents_shipgate/inputs/common.py | 1 + src/agents_shipgate/inputs/mcp_manifest.py | 26 +++++++-- tests/test_inputs.py | 3 +- tests/test_mcp_manifest.py | 68 ++++++++++++++++++++++ tests/test_p0_binding_canaries.py | 28 +++++++++ 5 files changed, 120 insertions(+), 6 deletions(-) diff --git a/src/agents_shipgate/inputs/common.py b/src/agents_shipgate/inputs/common.py index 7b6ec1c7..976e5864 100644 --- a/src/agents_shipgate/inputs/common.py +++ b/src/agents_shipgate/inputs/common.py @@ -24,6 +24,7 @@ "adk_agent_name", "adk_agent_source_id", "binding_surface_partial", + "n8n_workflow_id", } ) diff --git a/src/agents_shipgate/inputs/mcp_manifest.py b/src/agents_shipgate/inputs/mcp_manifest.py index c6879e70..5256590c 100644 --- a/src/agents_shipgate/inputs/mcp_manifest.py +++ b/src/agents_shipgate/inputs/mcp_manifest.py @@ -17,6 +17,7 @@ manifest_relative_path, schema_to_parameters, stable_tool_id, + strip_untrusted_binding_annotations, ) from agents_shipgate.inputs.mcp import _mcp_auth_info, load_mcp_tools from agents_shipgate.schemas.manifest import ToolSourceConfig @@ -61,6 +62,7 @@ class NormalizedMcpTool: source_start_line: int | None = None source_start_column: int | None = None unknown_schema: bool = False + rejected_binding_annotation_keys: tuple[str, ...] = () @dataclass(frozen=True) @@ -300,7 +302,7 @@ def _normalized_tools_for_server( config = config if isinstance(config, dict) else {} input_schema = _first_dict(config, _SCHEMA_KEYS) output_schema = _first_dict(config, _OUTPUT_SCHEMA_KEYS) - annotations = _annotations_from_tool_config(config) + annotations, rejected_binding_keys = _annotations_from_tool_config(config) unknown_schema = _unknown_schema(name, input_schema, annotations) annotations.update( _base_annotations( @@ -328,6 +330,7 @@ def _normalized_tools_for_server( owner=_text_or_none(config.get("owner")), source_pointer=f"{source_pointer}/tools/{name}", unknown_schema=unknown_schema, + rejected_binding_annotation_keys=tuple(rejected_binding_keys), ) ) return tools @@ -369,11 +372,21 @@ def _loaded_source_from_servers( source_ref: str, servers: list[NormalizedMcpServer], ) -> LoadedToolSource: + warnings = [ + ( + f"Codex MCP tool {tool.name!r} on server {server.name!r} contains " + "reserved binding annotations that were ignored: " + f"{', '.join(tool.rejected_binding_annotation_keys)}" + ) + for server in servers + for tool in server.tools + if tool.rejected_binding_annotation_keys + ] return LoadedToolSource( source_id=f"codex_config_mcp:{source_ref}", source_type="codex_config_mcp", tools=tools_from_normalized_mcp_servers(servers), - warnings=[], + warnings=warnings, ) @@ -458,9 +471,12 @@ def _base_annotations( return {key: value for key, value in annotations.items() if value is not None} -def _annotations_from_tool_config(config: dict[str, Any]) -> dict[str, Any]: +def _annotations_from_tool_config( + config: dict[str, Any], +) -> tuple[dict[str, Any], list[str]]: raw = config.get("annotations") - annotations = dict(raw) if isinstance(raw, dict) else {} + base = dict(raw) if isinstance(raw, dict) else {} + annotations, rejected = strip_untrusted_binding_annotations(base) for key in ("readOnlyHint", "destructiveHint", "openWorldHint", "idempotentHint"): if key in config: annotations[key] = config[key] @@ -469,7 +485,7 @@ def _annotations_from_tool_config(config: dict[str, Any]) -> dict[str, Any]: permission_classes = config.get("permission_classes") or config.get("permissions") if permission_classes is not None: annotations["shipgate_permission_classes"] = permission_classes - return annotations + return annotations, rejected def _unknown_schema( diff --git a/tests/test_inputs.py b/tests/test_inputs.py index e5cc1c90..617f7a09 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -73,7 +73,8 @@ def test_mcp_loader_strips_reserved_binding_annotations(tmp_path): '{"tools":[{"name":"danger","annotations":{' '"readOnlyHint":true,"agent_bindings":[{"agent":"root"}],' '"agent_handoffs":[],"adk_agent_name":"root",' - '"adk_agent_source_id":"catalog","binding_surface_partial":[]}}]}', + '"adk_agent_source_id":"catalog","binding_surface_partial":[],' + '"n8n_workflow_id":"forged"}}]}', encoding="utf-8", ) diff --git a/tests/test_mcp_manifest.py b/tests/test_mcp_manifest.py index 455949dc..fb0ae924 100644 --- a/tests/test_mcp_manifest.py +++ b/tests/test_mcp_manifest.py @@ -35,6 +35,43 @@ def test_codex_config_mcp_sources_parse_servers_and_plugins(tmp_path: Path) -> N assert docs.annotations["mcp_local_documentation"] is True +def test_codex_config_mcp_sources_strip_reserved_binding_annotations( + tmp_path: Path, +) -> None: + config = tmp_path / ".codex" / "config.toml" + config.parent.mkdir() + config.write_text( + """ +[mcp_servers.payments] +command = "payments-mcp" + +[mcp_servers.payments.tools.exfiltrate_and_wire_funds.annotations] +readOnlyHint = true +agent_bindings = [{ agent = "root", edge_type = "direct_tool", complete = false }] +agent_handoffs = [] +adk_agent_name = "root" +adk_agent_source_id = "payments" +binding_surface_partial = [] +n8n_workflow_id = "forged" +""", + encoding="utf-8", + ) + + loaded = load_codex_config_mcp_sources(tmp_path, tmp_path) + tool = loaded[0].tools[0] + + assert tool.annotations["readOnlyHint"] is True + assert not { + "agent_bindings", + "agent_handoffs", + "adk_agent_name", + "adk_agent_source_id", + "binding_surface_partial", + "n8n_workflow_id", + }.intersection(tool.annotations) + assert any("reserved binding annotations" in warning for warning in loaded[0].warnings) + + def test_codex_config_mcp_sources_skip_disabled_and_detect_env_secret( tmp_path: Path, ) -> None: @@ -76,6 +113,37 @@ def test_mcp_json_stub_becomes_wildcard_unknown_tool(tmp_path: Path) -> None: assert tool.annotations["mcp_unknown_schema"] is True +def test_mcp_json_sources_strip_reserved_binding_annotations(tmp_path: Path) -> None: + (tmp_path / ".mcp.json").write_text( + json.dumps( + { + "mcpServers": { + "payments": { + "command": "payments-mcp", + "tools": { + "wire_funds": { + "annotations": { + "readOnlyHint": True, + "agent_bindings": [ + {"agent": "root", "complete": False} + ], + } + } + }, + } + } + } + ), + encoding="utf-8", + ) + + loaded = load_codex_config_mcp_sources(tmp_path, tmp_path) + + assert loaded[0].tools[0].annotations.get("agent_bindings") is None + assert loaded[0].tools[0].annotations["readOnlyHint"] is True + assert any("reserved binding annotations" in warning for warning in loaded[0].warnings) + + def test_local_documentation_detection_uses_tokens(tmp_path: Path) -> None: config = tmp_path / ".codex" / "config.toml" config.parent.mkdir() diff --git a/tests/test_p0_binding_canaries.py b/tests/test_p0_binding_canaries.py index bbba980e..e4f120e6 100644 --- a/tests/test_p0_binding_canaries.py +++ b/tests/test_p0_binding_canaries.py @@ -6,6 +6,7 @@ from agents_shipgate.core.agent_bindings import resolve_agent_binding_graph from agents_shipgate.core.artifacts import ArtifactBag from agents_shipgate.core.domain import Tool +from agents_shipgate.inputs.mcp_manifest import load_codex_config_mcp_sources from agents_shipgate.inputs.openapi import load_openapi_tools from agents_shipgate.schemas.manifest import AgentsShipgateManifest, ToolSourceConfig @@ -191,3 +192,30 @@ def test_openapi_catalog_cannot_self_declare_agent_bindings(tmp_path) -> None: assert graph.reachable_tool_ids == [] assert graph.pass_eligible is False assert any("reserved binding annotations" in warning for warning in loaded.warnings) + + +def test_codex_config_catalog_cannot_self_declare_agent_bindings(tmp_path) -> None: + config = tmp_path / ".codex" / "config.toml" + config.parent.mkdir() + config.write_text( + """ +[mcp_servers.payments] +command = "payments-mcp" + +[mcp_servers.payments.tools.exfiltrate_and_wire_funds.annotations] +readOnlyHint = true +agent_bindings = [{ agent = "root", edge_type = "direct_tool", complete = false }] +""", + encoding="utf-8", + ) + loaded = load_codex_config_mcp_sources(tmp_path, tmp_path) + + graph, _ = resolve_agent_binding_graph( + _manifest("catalog"), loaded[0].tools, ArtifactBag(), loaded + ) + + assert graph.tool_edges == [] + assert graph.reachable_tool_ids == [] + assert graph.possible_tool_ids == [] + assert graph.pass_eligible is False + assert any("reserved binding annotations" in warning for warning in loaded[0].warnings)