From a48fd200fbc1b208eb5957d61091fa748dc4049f Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Fri, 17 Jul 2026 22:18:53 -0700 Subject: [PATCH 1/2] Model skill-only plugins as zero-callable roots --- docs/manifest-v0.1.md | 7 ++ docs/passed-verdict-contract.md | 9 ++ src/agents_shipgate/inputs/codex_plugin.py | 106 +++++++++++++++++ tests/test_codex_plugin.py | 129 ++++++++++++++++++++- tests/test_codex_plugin_launch_package.py | 15 +-- 5 files changed, 258 insertions(+), 8 deletions(-) diff --git a/docs/manifest-v0.1.md b/docs/manifest-v0.1.md index 9a7c327d..5741e10c 100644 --- a/docs/manifest-v0.1.md +++ b/docs/manifest-v0.1.md @@ -248,6 +248,13 @@ Only tools loaded from explicit MCP inventories enter `tool_inventory[]` with `source_type: codex_plugin_mcp_inventory`. Apps, hooks, skills, and MCP server declarations are reported under `codex_plugin_surface`, not as tools. +A fully parsed plugin package that contains one or more valid skills and no +apps, MCP servers, hooks, MCP inventories, unknown manifest keys, component +path issues, or source warnings establishes a structural package root with a +complete zero-callable surface. It does not need a reviewed empty +`agent_bindings` declaration. Any skipped, unknown, or degraded plugin input +invalidates that structural proof and remains fail-closed. + ## n8n n8n support is configured through the top-level `n8n:` block, not through diff --git a/docs/passed-verdict-contract.md b/docs/passed-verdict-contract.md index 9e14bf1c..5cc24d8a 100644 --- a/docs/passed-verdict-contract.md +++ b/docs/passed-verdict-contract.md @@ -56,6 +56,15 @@ capability facts contain only tools proven reachable from the root. Reviewed closed-world declarations live under `agent_bindings`; coding agents must not invent or auto-apply them. +One non-agent package case is structural rather than declared: a fully parsed, +warning-free skill-only Codex plugin can prove a complete package root with no +callable tools or handoffs. The compatibility projection currently represents +that package root in `binding_surface_facts.agents[]`, but it is not a runtime +agent and does not require a synthetic reviewed `agent_bindings` declaration. +Apps, MCP servers, hooks, MCP inventories, unknown manifest keys, skipped +entries, component path issues, or source warnings invalidate the zero-surface +proof. + The release decision also carries an explicit machine boundary: - `static_analysis_only: true`; diff --git a/src/agents_shipgate/inputs/codex_plugin.py b/src/agents_shipgate/inputs/codex_plugin.py index 6c942fe8..c4d8eea9 100644 --- a/src/agents_shipgate/inputs/codex_plugin.py +++ b/src/agents_shipgate/inputs/codex_plugin.py @@ -5,6 +5,7 @@ from agents_shipgate.core.artifact_models import CodexPluginArtifacts from agents_shipgate.core.domain import ( + AgentBindingObservation, LoadedToolSource, Tool, ) @@ -37,6 +38,19 @@ COMMAND_KEYS = {"command", "cmd", "run", "shell", "script"} PLUGIN_MANIFEST = ".codex-plugin/plugin.json" +_CODEX_PLUGIN_COMPONENT_KEYS = {"apps", "hooks", "mcpServers", "skills"} +_CODEX_PLUGIN_METADATA_KEYS = { + "author", + "description", + "homepage", + "interface", + "keywords", + "license", + "name", + "repository", + "version", +} +_KNOWN_CODEX_PLUGIN_KEYS = _CODEX_PLUGIN_COMPONENT_KEYS | _CODEX_PLUGIN_METADATA_KEYS def load_codex_plugin_artifacts( @@ -104,6 +118,16 @@ def load_codex_plugin_artifacts( artifacts.hook_stub_count = len(artifacts.hook_stubs) artifacts.mcp_inventory_file_count = len(artifacts.mcp_inventory_files) artifacts.warnings = sorted(dict.fromkeys(artifacts.warnings)) + if artifacts.warnings or artifacts.component_path_issues or any( + marketplace.skipped_entries for marketplace in artifacts.marketplaces + ): + # A package-root observation is a closed-world statement. If any part + # of the configured Codex plugin source was skipped or degraded, keep + # the binding graph incomplete instead of treating an observed skill + # as proof that the entire callable surface is empty. + for loaded_source in loaded_sources: + if loaded_source.source_type == "codex_plugin": + loaded_source.binding_observations = [] return loaded_sources, artifacts @@ -225,6 +249,12 @@ def _load_plugin_package( data, positions = load_structured_file_with_positions(manifest_path) if not isinstance(data, dict): raise InputParseError(f"Codex plugin manifest must contain an object: {manifest_path}") + unknown_keys = sorted(set(data) - _KNOWN_CODEX_PLUGIN_KEYS) + if unknown_keys: + artifacts.warnings.append( + "Codex plugin manifest contains unrecognized top-level keys " + f"{unknown_keys!r}; callable-surface completeness cannot be proven." + ) name = data.get("name") if isinstance(data.get("name"), str) else root.name source_id = f"codex_plugin:{source.id}/{name}" @@ -263,14 +293,90 @@ def _load_plugin_package( seen_roots[root_resolved] = plugin artifacts.plugins.append(plugin) + skill_start = len(artifacts.skills) + app_start = len(artifacts.apps) + mcp_start = len(artifacts.mcp_server_stubs) + hook_start = len(artifacts.hook_stubs) + issue_start = len(artifacts.component_path_issues) + warning_start = len(artifacts.warnings) + loaded_sources: list[LoadedToolSource] = [] _load_skills(data, root, base_dir, name, artifacts) _load_apps(data, root, base_dir, name, artifacts) loaded_sources.extend(_load_mcp_servers(data, root, base_dir, name, artifacts, inventories)) _load_hooks(data, root, base_dir, name, artifacts) + if _is_complete_skill_only_package( + data=data, + plugin=plugin, + skills=artifacts.skills[skill_start:], + apps=artifacts.apps[app_start:], + mcp_servers=artifacts.mcp_server_stubs[mcp_start:], + hooks=artifacts.hook_stubs[hook_start:], + component_path_issues=artifacts.component_path_issues[issue_start:], + warnings=artifacts.warnings[warning_start:], + has_declared_inventory=any( + inventory_plugin == name for inventory_plugin, _ in inventories + ), + ): + # Compatibility projection: the current binding schema names every + # graph root an "agent". A skill-only plugin is instead a package root + # whose fully parsed component graph proves that it exposes no + # callable tools or handoffs. No reviewed agent_bindings declaration + # is needed for this structural zero-capability fact. + loaded_sources.append( + LoadedToolSource( + source_id=source_id, + source_type="codex_plugin", + binding_observations=[ + AgentBindingObservation( + agent=f"codex-plugin:{name}", + source_id=source_id, + source=plugin.manifest_path, + source_pointer="", + tools_complete=True, + handoffs_complete=True, + ) + ], + ) + ) return loaded_sources +def _is_complete_skill_only_package( + *, + data: dict[str, Any], + plugin: CodexPluginSummary, + skills: list[CodexPluginSkillSummary], + apps: list[CodexPluginAppSummary], + mcp_servers: list[CodexPluginMcpServerStub], + hooks: list[CodexPluginHookStub], + component_path_issues: list[CodexPluginComponentPathIssue], + warnings: list[str], + has_declared_inventory: bool, +) -> bool: + unknown_keys = set(data) - _KNOWN_CODEX_PLUGIN_KEYS + non_skill_component_keys = ( + _CODEX_PLUGIN_COMPONENT_KEYS - {"skills"} + ) & set(data) + return bool(skills) and not any( + ( + unknown_keys, + non_skill_component_keys, + plugin.missing_fields, + plugin.name_mismatch, + plugin.duplicate_root, + plugin.duplicate_name, + apps, + mcp_servers, + hooks, + component_path_issues, + warnings, + has_declared_inventory, + any(skill.missing_fields or skill.duplicate for skill in skills), + ) + ) + + def _resolve_package_root( base_dir: Path, source_path: str, diff --git a/tests/test_codex_plugin.py b/tests/test_codex_plugin.py index 6fc2d248..179afd44 100644 --- a/tests/test_codex_plugin.py +++ b/tests/test_codex_plugin.py @@ -55,12 +55,101 @@ def test_codex_plugin_package_scan_keeps_non_tools_out_of_inventory( assert report.codex_plugin_surface.mcp_server_stub_count == 1 assert report.codex_plugin_surface.hook_stub_count == 1 assert report.tool_inventory == [] + assert report.binding_surface_facts.pass_eligible is False + assert {issue.kind for issue in report.binding_surface_facts.issues} == { + "ambiguous_root_agent" + } check_ids = {finding.check_id for finding in report.findings} assert "SHIP-INVENTORY-NOT-ENUMERABLE" not in check_ids assert "SHIP-CODEX-PLUGIN-MCP-SERVER-NOT-ENUMERABLE" in check_ids assert "SHIP-CODEX-PLUGIN-APP-SURFACE-NOT-ENUMERABLE" in check_ids +def test_unknown_codex_plugin_component_does_not_prove_empty_surface( + tmp_path: Path, +) -> None: + root = tmp_path / "plugins" / "skillish" + _write_skill_only_codex_plugin( + root, + extra_manifest={"futureRuntime": "./runtime.json"}, + ) + manifest = tmp_path / "shipgate.yaml" + manifest.write_text( + textwrap.dedent( + """ + version: "0.1" + project: + name: codex-plugin-test + agent: + name: codex-plugin-review + declared_purpose: + - review a plugin package + environment: + target: local + tool_sources: + - id: skillish + type: codex_plugin + mode: package + path: plugins/skillish + output: + packet: + enabled: false + """ + ), + encoding="utf-8", + ) + + report, _ = run_scan(config_path=manifest) + + assert report.release_decision.decision == "insufficient_evidence" + assert any("futureRuntime" in warning for warning in report.source_warnings) + assert report.binding_surface_facts.pass_eligible is False + assert {issue.kind for issue in report.binding_surface_facts.issues} == { + "ambiguous_root_agent" + } + + +def test_skill_only_codex_plugin_proves_empty_surface_structurally( + tmp_path: Path, +) -> None: + root = tmp_path / "plugins" / "skillish" + _write_skill_only_codex_plugin(root) + manifest = tmp_path / "shipgate.yaml" + manifest.write_text( + textwrap.dedent( + """ + version: "0.1" + project: + name: codex-plugin-test + agent: + name: codex-plugin-review + declared_purpose: + - review a plugin package + environment: + target: local + tool_sources: + - id: skillish + type: codex_plugin + mode: package + path: plugins/skillish + output: + packet: + enabled: false + """ + ), + encoding="utf-8", + ) + + report, _ = run_scan(config_path=manifest) + + assert report.release_decision.decision == "passed" + assert report.binding_surface_facts.status == "structural" + assert report.binding_surface_facts.pass_eligible is True + [root_agent] = report.binding_surface_facts.agents + assert root_agent.name == "codex-plugin:skillish" + assert report.binding_surface_facts.reachable_tool_ids == [] + + def test_codex_plugin_mcp_inventory_enumerates_tools(tmp_path: Path) -> None: _write_codex_plugin(tmp_path / "plugins" / "browserish", include_app=False) inventory = tmp_path / "inventories" / "browser-tools.json" @@ -178,7 +267,7 @@ def test_codex_plugin_marketplace_missing_policy_is_finding(tmp_path: Path) -> N def test_codex_plugin_manifest_file_path_warning_flows_to_report( tmp_path: Path, ) -> None: - _write_codex_plugin(tmp_path / "plugins" / "browserish", include_app=False) + _write_skill_only_codex_plugin(tmp_path / "plugins" / "browserish") manifest = tmp_path / "shipgate.yaml" manifest.write_text( textwrap.dedent( @@ -211,6 +300,11 @@ def test_codex_plugin_manifest_file_path_warning_flows_to_report( "prefer the plugin root directory" in warning for warning in report.source_warnings ) + assert report.release_decision.decision == "insufficient_evidence" + assert report.binding_surface_facts.pass_eligible is False + assert {issue.kind for issue in report.binding_surface_facts.issues} == { + "ambiguous_root_agent" + } def test_detect_and_init_route_codex_plugin_only_workspace(tmp_path: Path) -> None: @@ -353,3 +447,36 @@ def _write_codex_plugin(root: Path, *, include_app: bool) -> None: json.dumps({"preRun": {"command": "touch should-never-run"}}), encoding="utf-8", ) + + +def _write_skill_only_codex_plugin( + root: Path, + *, + extra_manifest: dict[str, object] | None = None, +) -> None: + (root / ".codex-plugin").mkdir(parents=True) + (root / "skills" / "review").mkdir(parents=True) + plugin: dict[str, object] = { + "name": root.name, + "version": "1.0.0", + "description": "Review a static skill-only Codex plugin.", + "skills": "./skills/", + } + plugin.update(extra_manifest or {}) + (root / ".codex-plugin" / "plugin.json").write_text( + json.dumps(plugin), + encoding="utf-8", + ) + (root / "skills" / "review" / "SKILL.md").write_text( + textwrap.dedent( + """ + --- + name: review + description: Review the local plugin without calling tools. + --- + + # Review + """ + ).lstrip(), + encoding="utf-8", + ) diff --git a/tests/test_codex_plugin_launch_package.py b/tests/test_codex_plugin_launch_package.py index 0639620c..9350e79a 100644 --- a/tests/test_codex_plugin_launch_package.py +++ b/tests/test_codex_plugin_launch_package.py @@ -90,13 +90,6 @@ def test_agents_shipgate_codex_plugin_scans_without_plugin_findings( type: codex_plugin mode: marketplace path: .agents/plugins/marketplace.json - agent_bindings: - declarations: - - agent: root - complete: true - tools: [] - handoffs: [] - reason: reviewed skill-only plugin has no callable tools output: packet: enabled: false @@ -117,6 +110,14 @@ def test_agents_shipgate_codex_plugin_scans_without_plugin_findings( assert report.codex_plugin_surface.mcp_server_stub_count == 0 assert report.codex_plugin_surface.hook_stub_count == 0 assert report.tool_inventory == [] + assert report.binding_surface_facts.status == "structural" + assert report.binding_surface_facts.pass_eligible is True + root = next( + agent + for agent in report.binding_surface_facts.agents + if agent.agent_id == report.binding_surface_facts.root_agent_id + ) + assert root.name == "codex-plugin:agents-shipgate" assert { finding.check_id for finding in report.findings From bea5c8ce320758a522ea16f8fa56fb38600c5075 Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Fri, 17 Jul 2026 22:28:13 -0700 Subject: [PATCH 2/2] Make verify help assertion ANSI-safe --- tests/test_verify_auto_base.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_verify_auto_base.py b/tests/test_verify_auto_base.py index b36c2cc4..11481195 100644 --- a/tests/test_verify_auto_base.py +++ b/tests/test_verify_auto_base.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import re import subprocess from pathlib import Path @@ -24,6 +25,12 @@ runner = CliRunner() +_ANSI_CSI = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +def _strip_ansi(text: str) -> str: + return _ANSI_CSI.sub("", text) + def _git(repo: Path, *args: str) -> None: subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True) @@ -301,9 +308,12 @@ def test_explicit_local_base_main_remains_supported(tmp_path: Path) -> None: def test_base_help_documents_remote_only_auto_detection() -> None: - result = runner.invoke(app, ["verify", "--help"]) + result = runner.invoke(app, ["verify", "--help"], color=True) assert result.exit_code == 0, result.output - assert "origin/HEAD, origin/main, origin/master" in result.output - assert "origin/main, origin/master, main, master" not in result.output - assert "Local main/master are used only" in result.output + normalized_output = " ".join( + _strip_ansi(result.output).replace("│", " ").split() + ) + assert "origin/HEAD, origin/main, origin/master" in normalized_output + assert "origin/main, origin/master, main, master" not in normalized_output + assert "Local main/master are used only" in normalized_output