From e65d984edc3e59524edc5bfc18242132c31ffee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=E1=BB=93=20Kh=E1=BA=AFc=20Huy?= <256174233+builtbyhuy@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:18:17 +0700 Subject: [PATCH 1/8] fix: support Claude-compatible marketplaces --- src/check-plugin.mjs | 49 +++++++++++++++++++++++++--- test/check-plugin.test.mjs | 67 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/src/check-plugin.mjs b/src/check-plugin.mjs index f1161d0..256cef6 100644 --- a/src/check-plugin.mjs +++ b/src/check-plugin.mjs @@ -1,5 +1,5 @@ import { spawn } from 'node:child_process'; -import { realpath } from 'node:fs/promises'; +import { readFile, realpath } from 'node:fs/promises'; import path from 'node:path'; import { AppServerClient } from './app-server-client.mjs'; import { createIsolation as createRealIsolation } from './isolation.mjs'; @@ -66,6 +66,44 @@ function pathIsWithin(candidate, root) { return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); } +export async function resolveMarketplaceManifestPath( + marketplaceRoot, + marketplaceName, + dependencies = {} +) { + const read = dependencies.readFile ?? readFile; + const canonicalize = dependencies.realpath ?? realpath; + const canonicalRoot = await canonicalize(marketplaceRoot); + let invalidManifest = false; + for (const relativePath of [ + path.join('.agents', 'plugins', 'marketplace.json'), + path.join('.claude-plugin', 'marketplace.json') + ]) { + const candidate = path.join(canonicalRoot, relativePath); + let manifest; + try { + manifest = JSON.parse(await read(candidate, 'utf8')); + } catch (cause) { + if (cause?.code === 'ENOENT') continue; + if (cause instanceof SyntaxError) { + invalidManifest = true; + continue; + } + throw cause; + } + if (manifest?.name !== marketplaceName) continue; + const canonicalPath = await canonicalize(candidate); + if (!pathIsWithin(canonicalPath, canonicalRoot)) { + throw new Error('Marketplace manifest escaped the marketplace root'); + } + return canonicalPath; + } + if (invalidManifest) { + throw new Error(`No valid marketplace manifest matched ${marketplaceName}`); + } + throw new Error(`No marketplace manifest matched ${marketplaceName}`); +} + function validatePluginReadIdentity(plugin, install, installed, marketplacePath) { const matches = plugin?.summary?.id === install.pluginId && plugin.summary.name === install.name && @@ -111,6 +149,12 @@ export async function checkPlugin(options, dependencies = {}) { ]))).stdout), marketplaceRoot ); + const findMarketplaceManifest = dependencies.resolveMarketplaceManifestPath ?? + resolveMarketplaceManifestPath; + const marketplacePath = await findMarketplaceManifest( + marketplaceRoot, + marketplace.marketplaceName + ); const install = JSON.parse((await runCommand(invocation([ 'plugin', 'add', options.plugin, '--marketplace', marketplace.marketplaceName, '--json' ]))).stdout); @@ -128,9 +172,6 @@ export async function checkPlugin(options, dependencies = {}) { timeoutMs: options.timeoutMs ?? 30_000 }); await client.initialize(); - const marketplacePath = path.join( - marketplaceRoot, '.agents', 'plugins', 'marketplace.json' - ); const pluginResult = await client.request('plugin/read', { pluginName: options.plugin, marketplacePath diff --git a/test/check-plugin.test.mjs b/test/check-plugin.test.mjs index e1d454d..51fecd2 100644 --- a/test/check-plugin.test.mjs +++ b/test/check-plugin.test.mjs @@ -1,10 +1,10 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtemp, rm, symlink } from 'node:fs/promises'; +import { mkdir, mkdtemp, realpath, rm, symlink, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { checkPlugin } from '../src/check-plugin.mjs'; +import { checkPlugin, resolveMarketplaceManifestPath } from '../src/check-plugin.mjs'; const fixtureRoot = fileURLToPath(new URL('./fixtures/marketplace', import.meta.url)); const pluginRoot = path.join(fixtureRoot, 'plugins', 'sample'); @@ -217,6 +217,69 @@ test('canonicalizes a symlink marketplace root across every Codex boundary', asy } }); +test('resolves the marketplace manifest Codex accepted without assuming an agents layout', async () => { + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-manifest-')); + const claudeDirectory = path.join(temporaryRoot, '.claude-plugin'); + const agentsDirectory = path.join(temporaryRoot, '.agents', 'plugins'); + const claudeManifest = path.join(claudeDirectory, 'marketplace.json'); + await mkdir(claudeDirectory, { recursive: true }); + await mkdir(agentsDirectory, { recursive: true }); + await writeFile( + path.join(agentsDirectory, 'marketplace.json'), + '{"name":"different-marketplace","plugins":[]}\n' + ); + await writeFile( + claudeManifest, + '{"name":"local-marketplace","plugins":[]}\n' + ); + try { + assert.equal( + await resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace'), + await realpath(claudeManifest) + ); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } +}); + +test('rejects a matching marketplace manifest symlink that escapes the checkout', async () => { + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-manifest-')); + const outsideRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-outside-')); + const outsideManifest = path.join(outsideRoot, 'marketplace.json'); + const agentsDirectory = path.join(temporaryRoot, '.agents', 'plugins'); + await mkdir(agentsDirectory, { recursive: true }); + await writeFile(outsideManifest, '{"name":"local-marketplace","plugins":[]}\n'); + await symlink(outsideManifest, path.join(agentsDirectory, 'marketplace.json')); + try { + await assert.rejects( + resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace'), + /escaped the marketplace root/ + ); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + await rm(outsideRoot, { recursive: true, force: true }); + } +}); + +test('requests plugin evidence from the resolved marketplace manifest', async () => { + const claudeManifest = path.join(fixtureRoot, '.claude-plugin', 'marketplace.json'); + const responses = successfulResponses(); + responses.plugin.plugin.marketplacePath = claudeManifest; + const observed = harness({ plugin: responses.plugin }); + observed.dependencies.resolveMarketplaceManifestPath = async (root, name) => { + assert.equal(root, fixtureRoot); + assert.equal(name, 'local-marketplace'); + return claudeManifest; + }; + + await checkPlugin(options, observed.dependencies); + + assert.deepEqual(observed.requests[0], [ + 'plugin/read', + { pluginName: 'sample', marketplacePath: claudeManifest } + ]); +}); + test('rejects install JSON that lacks Codex-owned install evidence', async () => { const observed = harness({ install: { name: 'sample', marketplaceName: 'local-marketplace' } }); From e03e3c919e3ac13949b71b7f4e26d99378ce532e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=E1=BB=93=20Kh=E1=BA=AFc=20Huy?= <256174233+builtbyhuy@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:22:58 +0700 Subject: [PATCH 2/8] fix: bind marketplace manifest precedence --- src/check-plugin.mjs | 35 +++++++++------- test/check-plugin.test.mjs | 81 ++++++++++++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 21 deletions(-) diff --git a/src/check-plugin.mjs b/src/check-plugin.mjs index 256cef6..1fa18b4 100644 --- a/src/check-plugin.mjs +++ b/src/check-plugin.mjs @@ -1,5 +1,5 @@ import { spawn } from 'node:child_process'; -import { readFile, realpath } from 'node:fs/promises'; +import { readFile, realpath, stat } from 'node:fs/promises'; import path from 'node:path'; import { AppServerClient } from './app-server-client.mjs'; import { createIsolation as createRealIsolation } from './isolation.mjs'; @@ -73,34 +73,41 @@ export async function resolveMarketplaceManifestPath( ) { const read = dependencies.readFile ?? readFile; const canonicalize = dependencies.realpath ?? realpath; + const inspect = dependencies.stat ?? stat; const canonicalRoot = await canonicalize(marketplaceRoot); - let invalidManifest = false; + // Keep this order aligned with core-plugins/src/marketplace.rs in Codex. for (const relativePath of [ path.join('.agents', 'plugins', 'marketplace.json'), - path.join('.claude-plugin', 'marketplace.json') + path.join('.agents', 'plugins', 'api_marketplace.json'), + path.join('.claude-plugin', 'marketplace.json'), + path.join('.cursor-plugin', 'marketplace.json') ]) { const candidate = path.join(canonicalRoot, relativePath); + let canonicalPath; + try { + canonicalPath = await canonicalize(candidate); + } catch (cause) { + if (cause?.code === 'ENOENT' || cause?.code === 'ENOTDIR') continue; + throw cause; + } + if (!pathIsWithin(canonicalPath, canonicalRoot)) { + throw new Error('Marketplace manifest escaped the marketplace root'); + } + if (!(await inspect(canonicalPath)).isFile()) continue; let manifest; try { - manifest = JSON.parse(await read(candidate, 'utf8')); + manifest = JSON.parse(await read(canonicalPath, 'utf8')); } catch (cause) { - if (cause?.code === 'ENOENT') continue; if (cause instanceof SyntaxError) { - invalidManifest = true; - continue; + throw new Error('Selected marketplace manifest was not valid JSON'); } throw cause; } - if (manifest?.name !== marketplaceName) continue; - const canonicalPath = await canonicalize(candidate); - if (!pathIsWithin(canonicalPath, canonicalRoot)) { - throw new Error('Marketplace manifest escaped the marketplace root'); + if (manifest?.name !== marketplaceName) { + throw new Error('Selected marketplace manifest identity did not match Codex marketplace output'); } return canonicalPath; } - if (invalidManifest) { - throw new Error(`No valid marketplace manifest matched ${marketplaceName}`); - } throw new Error(`No marketplace manifest matched ${marketplaceName}`); } diff --git a/test/check-plugin.test.mjs b/test/check-plugin.test.mjs index 51fecd2..a540933 100644 --- a/test/check-plugin.test.mjs +++ b/test/check-plugin.test.mjs @@ -217,31 +217,91 @@ test('canonicalizes a symlink marketplace root across every Codex boundary', asy } }); -test('resolves the marketplace manifest Codex accepted without assuming an agents layout', async () => { +test('resolves a Claude-compatible marketplace when earlier Codex layouts are absent', async () => { const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-manifest-')); const claudeDirectory = path.join(temporaryRoot, '.claude-plugin'); - const agentsDirectory = path.join(temporaryRoot, '.agents', 'plugins'); const claudeManifest = path.join(claudeDirectory, 'marketplace.json'); await mkdir(claudeDirectory, { recursive: true }); + await writeFile( + claudeManifest, + '{"name":"local-marketplace","plugins":[]}\n' + ); + try { + assert.equal( + await resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace'), + await realpath(claudeManifest) + ); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } +}); + +test('mirrors released Codex marketplace-layout precedence when manifests overlap', async () => { + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-manifest-')); + const agentsDirectory = path.join(temporaryRoot, '.agents', 'plugins'); + const claudeDirectory = path.join(temporaryRoot, '.claude-plugin'); + const agentsManifest = path.join(agentsDirectory, 'marketplace.json'); await mkdir(agentsDirectory, { recursive: true }); + await mkdir(claudeDirectory, { recursive: true }); + await writeFile(agentsManifest, '{"name":"local-marketplace","plugins":[]}\n'); + await writeFile( + path.join(claudeDirectory, 'marketplace.json'), + '{"name":"local-marketplace","plugins":[]}\n' + ); + try { + assert.equal( + await resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace'), + await realpath(agentsManifest) + ); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } +}); + +test('rejects identity drift in the first marketplace layout Codex would select', async () => { + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-manifest-')); + const agentsDirectory = path.join(temporaryRoot, '.agents', 'plugins'); + const claudeDirectory = path.join(temporaryRoot, '.claude-plugin'); + await mkdir(agentsDirectory, { recursive: true }); + await mkdir(claudeDirectory, { recursive: true }); await writeFile( path.join(agentsDirectory, 'marketplace.json'), '{"name":"different-marketplace","plugins":[]}\n' ); await writeFile( - claudeManifest, + path.join(claudeDirectory, 'marketplace.json'), '{"name":"local-marketplace","plugins":[]}\n' ); try { - assert.equal( - await resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace'), - await realpath(claudeManifest) + await assert.rejects( + resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace'), + /identity did not match Codex marketplace output/ ); } finally { await rm(temporaryRoot, { recursive: true, force: true }); } }); +test('supports Codex API and Cursor-compatible marketplace layouts', async () => { + for (const relativePath of [ + path.join('.agents', 'plugins', 'api_marketplace.json'), + path.join('.cursor-plugin', 'marketplace.json') + ]) { + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-manifest-')); + const manifestPath = path.join(temporaryRoot, relativePath); + await mkdir(path.dirname(manifestPath), { recursive: true }); + await writeFile(manifestPath, '{"name":"local-marketplace","plugins":[]}\n'); + try { + assert.equal( + await resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace'), + await realpath(manifestPath) + ); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } + } +}); + test('rejects a matching marketplace manifest symlink that escapes the checkout', async () => { const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-manifest-')); const outsideRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-outside-')); @@ -250,11 +310,18 @@ test('rejects a matching marketplace manifest symlink that escapes the checkout' await mkdir(agentsDirectory, { recursive: true }); await writeFile(outsideManifest, '{"name":"local-marketplace","plugins":[]}\n'); await symlink(outsideManifest, path.join(agentsDirectory, 'marketplace.json')); + let reads = 0; try { await assert.rejects( - resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace'), + resolveMarketplaceManifestPath(temporaryRoot, 'local-marketplace', { + async readFile() { + reads += 1; + throw new Error('outside manifest must not be read'); + } + }), /escaped the marketplace root/ ); + assert.equal(reads, 0); } finally { await rm(temporaryRoot, { recursive: true, force: true }); await rm(outsideRoot, { recursive: true, force: true }); From c0b579b9fa8f28b8ee896b99a394e470f6c1fb2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=E1=BB=93=20Kh=E1=BA=AFc=20Huy?= <256174233+builtbyhuy@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:26:11 +0700 Subject: [PATCH 3/8] ci: add public fixture workflow --- .github/workflows/public-fixtures.yml | 56 +++++++++++++++++++++++++++ test/workflow-contract.test.mjs | 53 +++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 .github/workflows/public-fixtures.yml diff --git a/.github/workflows/public-fixtures.yml b/.github/workflows/public-fixtures.yml new file mode 100644 index 0000000..f34389d --- /dev/null +++ b/.github/workflows/public-fixtures.yml @@ -0,0 +1,56 @@ +name: Public plugin fixture matrix + +on: + push: + branches: + - main + paths: + - .github/workflows/public-fixtures.yml + - .dockerignore + - Dockerfile + - package.json + - scripts/falsify-public-fixtures.mjs + - src/** + - test/public-fixtures.test.mjs + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: public-fixtures-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + strict-public-fixtures: + name: Ten public plugins on two released versions + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + CODEX_CURRENT_VERSION: 0.147.0 + CODEX_PRIOR_VERSION: 0.146.1 + CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: artifacts/public-fixtures + steps: + - name: Check out source without credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Set up Node 24 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24.19.0 + + - name: Falsify exact public plugin fixtures + run: npm run falsify:public + + - name: Upload public-fixture evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: public-fixture-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/public-fixtures + if-no-files-found: error + retention-days: 14 + compression-level: 9 + include-hidden-files: false diff --git a/test/workflow-contract.test.mjs b/test/workflow-contract.test.mjs index 3115c53..7ae06ce 100644 --- a/test/workflow-contract.test.mjs +++ b/test/workflow-contract.test.mjs @@ -141,3 +141,56 @@ test('strict workflow lets the falsifier exclusively create its evidence directo [] ); }); + +test('public fixture matrix runs only from trusted main pushes or manual dispatch', async () => { + const source = await workflow('public-fixtures.yml'); + + assert.deepEqual(mappingKeys(section(source, 'on'), 2), ['push', 'workflow_dispatch']); + assert.deepEqual(section(source, 'permissions').filter((line) => line.trim()), [ + ' contents: read' + ]); + assert.match(source, /^ runs-on: ubuntu-24\.04$/mu); + assert.match(source, /^ timeout-minutes: 60$/mu); + assert.match(source, /^ node-version: 24\.19\.0$/mu); + assert.match(source, /^ CODEX_CURRENT_VERSION: 0\.147\.0$/mu); + assert.match(source, /^ CODEX_PRIOR_VERSION: 0\.146\.1$/mu); + assert.match( + source, + /^ CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: artifacts\/public-fixtures$/mu + ); + assert.doesNotMatch(source, /^\s*pull_request:/mu); + assert.doesNotMatch(source, /^\s*run: npm test$/mu); + assert.equal(source.match(/^\s*run: npm run falsify:public$/gmu)?.length, 1); + assert.equal( + scalar(stepUsing(source, CHECKOUT).source, 'persist-credentials', 10), + 'false' + ); + assert.ok(stepUsing(source, SETUP_NODE)); + const upload = stepUsing(source, UPLOAD); + assert.equal(scalar(upload.source, 'path', 10), 'artifacts/public-fixtures'); + assert.equal(scalar(upload.source, 'if-no-files-found', 10), 'error'); + assert.ok(upload.source.includes(' if: always()')); + assert.deepEqual( + steps(source).filter((step) => step.source.some((line) => line.includes('uses:'))).length, + 3 + ); +}); + +test('public fixture runner exclusively creates its evidence directory', async () => { + const source = await workflow('public-fixtures.yml'); + const workflowSteps = steps(source); + const runnerIndex = workflowSteps.findIndex((step) => ( + step.source.includes(' run: npm run falsify:public') + )); + + assert.notEqual(runnerIndex, -1); + assert.deepEqual( + workflowSteps + .slice(0, runnerIndex) + .filter((step) => step.source.some((line) => ( + line.includes('artifacts/public-fixtures') + ))) + .map((step) => step.name), + [] + ); +}); From a854579dc574555f55ce7a5f080404e730b96712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=E1=BB=93=20Kh=E1=BA=AFc=20Huy?= <256174233+builtbyhuy@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:27:29 +0700 Subject: [PATCH 4/8] docs: define public fixture matrix --- docs/evidence/public-fixture-matrix.md | 65 +++++++++++++++++++ .../plans/2026-08-10-codex-plugin-check.md | 63 ++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 docs/evidence/public-fixture-matrix.md diff --git a/docs/evidence/public-fixture-matrix.md b/docs/evidence/public-fixture-matrix.md new file mode 100644 index 0000000..5523fc8 --- /dev/null +++ b/docs/evidence/public-fixture-matrix.md @@ -0,0 +1,65 @@ +# Public plugin fixture matrix evidence + +## Decision + +`HOLD`. The immutable ten-repository input audit is complete, but the strict +Linux matrix has not run yet. This document records inputs and observed output; +it does not turn a static compatibility expectation into a runtime result. + +## Fixed inputs + +Each repository is fetched at one full commit SHA without credentials. The +target matrix is Codex `0.147.0` and `0.146.1`, for 20 independent strict cells. + +| Repository | Commit | Marketplace root | Plugin | Preparation | Expected kinds | License | Strict result | +| --- | --- | --- | --- | --- | --- | --- | --- | +| [`bitrouter/bitrouter`](https://github.com/bitrouter/bitrouter) | [`678384888b73fc290ce4ce503a8a7f2a5cbf6da8`](https://github.com/bitrouter/bitrouter/commit/678384888b73fc290ce4ce503a8a7f2a5cbf6da8) | `.` | `bitrouter` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | +| [`Cassette-Editor/oh-my-cassette`](https://github.com/Cassette-Editor/oh-my-cassette) | [`cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32`](https://github.com/Cassette-Editor/oh-my-cassette/commit/cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32) | `.` | `oh-my-cassette` | `STATIC_ADAPTER:local-source-v1` | skill, MCP | MIT | `UNRUN` | +| [`mostlyharmless-ai/watercooler`](https://github.com/mostlyharmless-ai/watercooler) | [`a5efa89df02e7796e20881fef4847f129d84d367`](https://github.com/mostlyharmless-ai/watercooler/commit/a5efa89df02e7796e20881fef4847f129d84d367) | `.` | `watercooler` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | +| [`commercetools/commercetools-ai-plugins`](https://github.com/commercetools/commercetools-ai-plugins) | [`440d6bd56eb2969b6a0dd41e3fcff286def0787c`](https://github.com/commercetools/commercetools-ai-plugins/commit/440d6bd56eb2969b6a0dd41e3fcff286def0787c) | `.` | `commercetools` | `DIRECT` | skill, MCP | CC-BY-4.0 | `UNRUN` | +| [`agentis-tools/ctx`](https://github.com/agentis-tools/ctx) | [`1782436e0ebf8d95ef4c086d94351698c464c4ee`](https://github.com/agentis-tools/ctx/commit/1782436e0ebf8d95ef4c086d94351698c464c4ee) | `plugins/codex/ctx` | `ctx` | `DIRECT` | skill, hook | Apache-2.0 OR MIT | `UNRUN` | +| [`agentmail-to/agentmail-plugins`](https://github.com/agentmail-to/agentmail-plugins) | [`134887caf9375229415e09c760ae31baa4cc1ec3`](https://github.com/agentmail-to/agentmail-plugins/commit/134887caf9375229415e09c760ae31baa4cc1ec3) | `.` | `agentmail` | `DIRECT` | skill, MCP | MIT | `UNRUN` | +| [`ujjwalredd/sarathi`](https://github.com/ujjwalredd/sarathi) | [`08a51154a2f30af3eb4f6acb11115b9db912c5f8`](https://github.com/ujjwalredd/sarathi/commit/08a51154a2f30af3eb4f6acb11115b9db912c5f8) | `.` | `sarathi` | `DIRECT` | skill | MIT | `UNRUN` | +| [`sofus-nl/cc-plugin-codex`](https://github.com/sofus-nl/cc-plugin-codex) | [`cc5123f7fa18db9c38f838a9b70119e5a0a6847c`](https://github.com/sofus-nl/cc-plugin-codex/commit/cc5123f7fa18db9c38f838a9b70119e5a0a6847c) | `.` | `cc-plugin-codex` | `DIRECT` | skill, hook | Apache-2.0 + NOTICE | `UNRUN` | +| [`RMI/speedy-skills`](https://github.com/RMI/speedy-skills) | [`e983f800056a12b63fd60d5148538f98aaafe643`](https://github.com/RMI/speedy-skills/commit/e983f800056a12b63fd60d5148538f98aaafe643) | `.` | `example-minimal` | `DIRECT` | skill | MIT | `UNRUN` | +| [`roadrunner-tuff/roadrunner-admin-plugin`](https://github.com/roadrunner-tuff/roadrunner-admin-plugin) | [`8e130c07656c8f9db8bf5431332c9aed60a4b133`](https://github.com/roadrunner-tuff/roadrunner-admin-plugin/commit/8e130c07656c8f9db8bf5431332c9aed60a4b133) | `.` | `roadrunner-admin` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | + +`local-source-v1` may change only `/plugins/0/source` in +`.agents/plugins/marketplace.json` to `{"source":"local","path":"./"}`. +The evidence ledger must retain the original and adapted SHA-256 values. No +other fixture receives a rewrite. + +## Safety and evidence boundary + +- The runner must extract immutable commit archives into owned temporary state + with traversal and symlink escape checks. +- It must never execute fixture code, scripts, builds, package managers, hooks, + MCP servers, apps, authentication flows, or models. +- Every Codex probe must use the existing read-only, network-denied, + host-state-denied strict boundary. +- The retained artifact may contain source/tree/adapter hashes, relative + receipt names, sanitized receipts, and a summary. It must contain no upstream + source. +- A truthful plugin receipt may be `FAIL`. A fetch, isolation, tool, identity, + privacy, or ledger error must fail the matrix and cannot be relabeled as a + compatibility result. + +## Non-certifying diagnostics + +Before the fixed runner existed, direct environment probes observed both target +Codex versions successfully load `bitrouter/bitrouter`. The same probes exposed +a checker defect on the unmodified `.claude-plugin` marketplace in +`RMI/speedy-skills`; Codex itself installed it, while the checker incorrectly +required an `.agents` manifest. The checker now follows Codex's manifest +precedence, and the unmodified fixture passes on both versions. + +These environment observations helped falsify the implementation. They do not +satisfy the strict 10-repository gate. + +## Publication gate + +Change this decision from `HOLD` only after all 20 strict cells are retained, +validated against their exact repository/plugin/version identities, scanned +for private paths and execution sentinels, and independently reconciled with +the immutable inputs above. Stars, a green workflow, or a partial matrix do not +substitute for those receipts. diff --git a/docs/superpowers/plans/2026-08-10-codex-plugin-check.md b/docs/superpowers/plans/2026-08-10-codex-plugin-check.md index 5b07803..f6663be 100644 --- a/docs/superpowers/plans/2026-08-10-codex-plugin-check.md +++ b/docs/superpowers/plans/2026-08-10-codex-plugin-check.md @@ -471,6 +471,69 @@ git add .github README.md LICENSE SECURITY.md CONTRIBUTING.md docs/receipt.schem git commit -m "docs: prepare evidence-bound v0 release" ``` +### Task 7A: Exact public-fixture compatibility matrix + +**Files:** +- Create: `scripts/falsify-public-fixtures.mjs` +- Create: `test/public-fixtures.test.mjs` +- Create: `.github/workflows/public-fixtures.yml` +- Create: `docs/evidence/public-fixture-matrix.md` +- Modify: `package.json` +- Modify: `test/workflow-contract.test.mjs` + +**Interfaces:** +- The matrix is fixed to the ten immutable repository commits audited in + `PUBLIC_FIXTURE_AUDIT.md` and exact Codex `0.147.0` plus `0.146.1`. +- Nine fixtures are byte-for-byte `DIRECT`. `oh-my-cassette` alone uses the + bounded `local-source-v1` JSON adapter, replacing only + `/plugins/0/source` with a local source and recording before/after hashes. +- Public source is fetched without credentials into owned temporary state. + No repository script, build, dependency install, hook, MCP server, app, or + model is executed. No upstream source file is published in the artifact. + +- [ ] **Step 1: Write failing safety, identity, adapter, and ledger tests** + +Bind exactly ten unique HTTPS GitHub repositories, full commit SHAs, expected +plugin IDs, marketplace roots, licenses, and adapter IDs. Reject mutable refs, +path escapes, duplicate rows, extra adapter fields, untrusted workflow events, +receipt identity drift, missing expected capability kinds, absolute evidence +paths, and arbitrary tool errors mislabeled as incompatibility. + +- [ ] **Step 2: Verify RED** + +Run: `node --test test/public-fixtures.test.mjs test/workflow-contract.test.mjs` + +Expected: FAIL because the fixed runner and trusted workflow do not exist. + +- [ ] **Step 3: Implement the bounded fixed runner** + +Fetch and verify each exact commit with an isolated Git configuration, remove +only owned Git metadata before probing, preserve upstream license/notice files, +apply the single audited static adapter, and run the production strict CLI for +all 20 repository/version cells. Validate every receipt with the production +schema and exact source/plugin/platform/isolation expectations. A conformance +receipt may truthfully be `FAIL`; an unexpected tool error fails the matrix. + +- [ ] **Step 4: Run non-certifying local environment probes** + +Use the prepared released `0.147.0` and `0.146.1` binaries to reconcile each +observed capability set against the immutable source before strict CI. Record +these only as diagnostics; they cannot satisfy the strict gate. + +- [ ] **Step 5: Run the strict matrix in the private remote** + +The workflow is main-push/manual only, least privilege, Node 24, pinned Actions, +and always uploads a relative-path evidence ledger. Observe all 20 cells inside +the existing network/host-state-denied boundary. Retain source/tree/adapter +hashes, sanitized receipts, run SHA/URL, and artifact identity. + +- [ ] **Step 6: Reconcile and review before publication** + +Independently compare the artifact to immutable source expectations. Mark the +technical public-fixture gate `PASS` only when all ten fixtures have complete +observations or an evidence-backed version/API incompatibility classification. +Keep project/publication `HOLD` for any unexplained error or missing row. + ### Task 8: Bounded maintainer validation and application hold **Files:** From b6683c582c4261c02776a0d70ea14e91d16c293c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=E1=BB=93=20Kh=E1=BA=AFc=20Huy?= <256174233+builtbyhuy@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:38:57 +0700 Subject: [PATCH 5/8] docs: define application readiness gates --- docs/evidence/application-readiness.md | 82 ++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/evidence/application-readiness.md diff --git a/docs/evidence/application-readiness.md b/docs/evidence/application-readiness.md new file mode 100644 index 0000000..64c1510 --- /dev/null +++ b/docs/evidence/application-readiness.md @@ -0,0 +1,82 @@ +# Codex for Open Source application readiness + +## Decision + +`HOLD — NOT READY TO SUBMIT`. + +The repository is still private, the public-fixture runtime gate is incomplete, +and there is no independent retained usage yet. Submitting now would establish +that Huy created a project, but it would not establish the usage, ecosystem +importance, or ongoing maintainer responsibility that OpenAI asks applicants +to explain. + +OpenAI's current form says eligible applicants maintain active open-source +projects and that reviewers consider meaningful usage, broad adoption or clear +ecosystem importance, plus evidence of active maintenance such as PR review, +issue triage and release management. There is no published star threshold. +Source: [Codex for Open Source](https://openai.com/form/codex-for-oss/). + +## Current evidence + +| Gate | Current state | Required evidence | +| --- | --- | --- | +| Public repository and profile | `HOLD` | Public GitHub profile and public repository URL | +| Useful, bounded technical job | `PASS` | Real released-Codex loader evidence with no model/auth dependency | +| Synthetic strict boundary | `PASS` | Private Linux receipts for both target releases | +| Public-plugin compatibility | `UNRUN` | Reconciled 10-repository, 20-cell strict matrix | +| Independent retained use | `0` | External repositories keep the workflow enabled | +| Unknown regression caught | `1 local defect` | Publicly linkable regression/issue accepted or reproduced externally | +| Releases | `0 public releases` | Maintained tagged releases and release notes | +| Ongoing maintainer duties | `INSUFFICIENT` | Public issue triage, PR review and release work over time | +| Application | `NOT SUBMITTED` | All required form fields bound to public evidence | + +The local defect was real: an unmodified Claude-compatible marketplace loaded +in Codex while the checker rejected it. The fix is useful product evidence, but +it is not independent adoption because Huy found it while testing his own tool. + +## Conservative internal submission gates + +These are project safeguards, not claims that OpenAI publishes numeric rules. +Submit only after all are observed: + +1. The technical 10-repository matrix passes or honestly records and explains + every version/API incompatibility, with no unexplained tool error. +2. The repository is public with a signed-off `v0.1.0` release, public CI and + reproducible evidence links. +3. At least three unrelated plugin repositories retain the check in their + default-branch workflow. +4. The retained integrations remain green across two consecutive Codex release + updates, or one catches a real regression that an upstream maintainer accepts + or independently reproduces. +5. At least 60 days of public maintenance evidence exists, including issue + triage, review of an external contribution, and two release decisions. +6. Every application claim fits one of the public links in this directory; no + stars, downloads, adoption or maintainer duty is inferred from local work. + +## Exact form payload still required + +The current form requests: + +- first and last name; +- the email associated with the applicant's ChatGPT account; +- public GitHub username and public repository URL; +- primary or core maintainer role; +- a maximum-500-character qualification explanation; +- optional interest in Codex Security and API credits; +- OpenAI organization ID and a maximum-500-character API-credit use case; +- an optional maximum-500-character final note. + +Email and OpenAI organization ID are identity-linked inputs. They will be bound +at the submission boundary and never inferred from credentials or unrelated +local files. + +## Next evidence-producing actions + +1. Finish and privately reconcile the strict public-fixture matrix. +2. Publish only after security/code review and all technical gates pass. +3. Offer a bounded integration to maintainers whose immutable fixtures were + tested; do not open promotional issues or claim endorsement. +4. Record retained workflows, maintainer replies, defects and release upkeep in + the maintainer-validation ledger. +5. Re-evaluate the internal submission gates monthly. A green local build or a + star count alone never changes this decision. From 1a35b2c4a09be048a92ee01d90e0470fedd6c7f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=E1=BB=93=20Kh=E1=BA=AFc=20Huy?= <256174233+builtbyhuy@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:44:39 +0700 Subject: [PATCH 6/8] test: add public fixture matrix --- package.json | 1 + scripts/falsify-public-fixtures.mjs | 1434 +++++++++++++++++++++++++++ test/public-fixtures.test.mjs | 1095 ++++++++++++++++++++ 3 files changed, 2530 insertions(+) create mode 100644 scripts/falsify-public-fixtures.mjs create mode 100644 test/public-fixtures.test.mjs diff --git a/package.json b/package.json index 5ea9d4a..df9bbb9 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ }, "scripts": { "falsify": "node scripts/falsify-released-codex.mjs", + "falsify:public": "node scripts/falsify-public-fixtures.mjs", "test": "node --test" } } diff --git a/scripts/falsify-public-fixtures.mjs b/scripts/falsify-public-fixtures.mjs new file mode 100644 index 0000000..34fda67 --- /dev/null +++ b/scripts/falsify-public-fixtures.mjs @@ -0,0 +1,1434 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { createHash, randomUUID } from 'node:crypto'; +import { constants as fsConstants, realpathSync } from 'node:fs'; +import { + link, + lstat, + mkdtemp, + mkdir, + open, + readFile, + readdir, + readlink, + realpath, + rename, + rm, + symlink, + unlink, + writeFile +} from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { gunzipSync } from 'node:zlib'; + +import { main as cliMainReal, validateReceipt } from '../src/cli.mjs'; + +export const CODEX_VERSIONS = Object.freeze(['0.147.0', '0.146.1']); + +const FIXED_FIXTURES = [ + { + evidenceId: 'bitrouter', + repository: 'bitrouter/bitrouter', + repositoryUrl: 'https://github.com/bitrouter/bitrouter', + commit: '678384888b73fc290ce4ce503a8a7f2a5cbf6da8', + archiveUrl: 'https://codeload.github.com/bitrouter/bitrouter/tar.gz/678384888b73fc290ce4ce503a8a7f2a5cbf6da8', + archiveRoot: 'bitrouter-678384888b73fc290ce4ce503a8a7f2a5cbf6da8', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: '.', + plugin: 'bitrouter', + marketplace: 'bitrouter', + license: 'Apache-2.0', + licensePaths: ['LICENSE'], + expectedKinds: ['skill', 'mcp'] + }, + { + evidenceId: 'oh-my-cassette', + repository: 'Cassette-Editor/oh-my-cassette', + repositoryUrl: 'https://github.com/Cassette-Editor/oh-my-cassette', + commit: 'cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32', + archiveUrl: 'https://codeload.github.com/Cassette-Editor/oh-my-cassette/tar.gz/cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32', + archiveRoot: 'oh-my-cassette-cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32', + classification: 'STATIC_ADAPTER', + adapterId: 'local-source-v1', + adapterPath: '.agents/plugins/marketplace.json', + marketplaceRoot: '.', + plugin: 'oh-my-cassette', + marketplace: 'cassette-editor', + license: 'MIT', + licensePaths: ['LICENSE'], + expectedKinds: ['skill', 'mcp'] + }, + { + evidenceId: 'watercooler', + repository: 'mostlyharmless-ai/watercooler', + repositoryUrl: 'https://github.com/mostlyharmless-ai/watercooler', + commit: 'a5efa89df02e7796e20881fef4847f129d84d367', + archiveUrl: 'https://codeload.github.com/mostlyharmless-ai/watercooler/tar.gz/a5efa89df02e7796e20881fef4847f129d84d367', + archiveRoot: 'watercooler-a5efa89df02e7796e20881fef4847f129d84d367', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: '.', + plugin: 'watercooler', + marketplace: 'watercooler', + license: 'Apache-2.0', + licensePaths: ['LICENSE'], + expectedKinds: ['skill', 'mcp'] + }, + { + evidenceId: 'commercetools', + repository: 'commercetools/commercetools-ai-plugins', + repositoryUrl: 'https://github.com/commercetools/commercetools-ai-plugins', + commit: '440d6bd56eb2969b6a0dd41e3fcff286def0787c', + archiveUrl: 'https://codeload.github.com/commercetools/commercetools-ai-plugins/tar.gz/440d6bd56eb2969b6a0dd41e3fcff286def0787c', + archiveRoot: 'commercetools-ai-plugins-440d6bd56eb2969b6a0dd41e3fcff286def0787c', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: '.', + plugin: 'commercetools', + marketplace: 'commercetools', + license: 'CC-BY-4.0', + licensePaths: ['LICENSE'], + expectedKinds: ['skill', 'mcp'] + }, + { + evidenceId: 'ctx', + repository: 'agentis-tools/ctx', + repositoryUrl: 'https://github.com/agentis-tools/ctx', + commit: '1782436e0ebf8d95ef4c086d94351698c464c4ee', + archiveUrl: 'https://codeload.github.com/agentis-tools/ctx/tar.gz/1782436e0ebf8d95ef4c086d94351698c464c4ee', + archiveRoot: 'ctx-1782436e0ebf8d95ef4c086d94351698c464c4ee', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: 'plugins/codex/ctx', + plugin: 'ctx', + marketplace: 'ctx-local', + license: 'Apache-2.0 OR MIT', + licensePaths: ['LICENSE-APACHE', 'LICENSE-MIT'], + expectedKinds: ['skill', 'hook'] + }, + { + evidenceId: 'agentmail', + repository: 'agentmail-to/agentmail-plugins', + repositoryUrl: 'https://github.com/agentmail-to/agentmail-plugins', + commit: '134887caf9375229415e09c760ae31baa4cc1ec3', + archiveUrl: 'https://codeload.github.com/agentmail-to/agentmail-plugins/tar.gz/134887caf9375229415e09c760ae31baa4cc1ec3', + archiveRoot: 'agentmail-plugins-134887caf9375229415e09c760ae31baa4cc1ec3', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: '.', + plugin: 'agentmail', + marketplace: 'agentmail', + license: 'MIT', + licensePaths: ['LICENSE'], + expectedKinds: ['skill', 'mcp'] + }, + { + evidenceId: 'sarathi', + repository: 'ujjwalredd/sarathi', + repositoryUrl: 'https://github.com/ujjwalredd/sarathi', + commit: '08a51154a2f30af3eb4f6acb11115b9db912c5f8', + archiveUrl: 'https://codeload.github.com/ujjwalredd/sarathi/tar.gz/08a51154a2f30af3eb4f6acb11115b9db912c5f8', + archiveRoot: 'sarathi-08a51154a2f30af3eb4f6acb11115b9db912c5f8', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: '.', + plugin: 'sarathi', + marketplace: 'sarathi', + license: 'MIT', + licensePaths: ['LICENSE'], + expectedKinds: ['skill'] + }, + { + evidenceId: 'cc-plugin-codex', + repository: 'sofus-nl/cc-plugin-codex', + repositoryUrl: 'https://github.com/sofus-nl/cc-plugin-codex', + commit: 'cc5123f7fa18db9c38f838a9b70119e5a0a6847c', + archiveUrl: 'https://codeload.github.com/sofus-nl/cc-plugin-codex/tar.gz/cc5123f7fa18db9c38f838a9b70119e5a0a6847c', + archiveRoot: 'cc-plugin-codex-cc5123f7fa18db9c38f838a9b70119e5a0a6847c', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: '.', + plugin: 'cc-plugin-codex', + marketplace: 'cc-plugin-codex', + license: 'Apache-2.0', + licensePaths: [ + 'LICENSE', + 'plugins/cc-plugin-codex/LICENSE', + 'plugins/cc-plugin-codex/NOTICE' + ], + expectedKinds: ['skill', 'hook'] + }, + { + evidenceId: 'speedy-skills', + repository: 'RMI/speedy-skills', + repositoryUrl: 'https://github.com/RMI/speedy-skills', + commit: 'e983f800056a12b63fd60d5148538f98aaafe643', + archiveUrl: 'https://codeload.github.com/RMI/speedy-skills/tar.gz/e983f800056a12b63fd60d5148538f98aaafe643', + archiveRoot: 'speedy-skills-e983f800056a12b63fd60d5148538f98aaafe643', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: '.', + plugin: 'example-minimal', + marketplace: 'speedy-skills', + license: 'MIT', + licensePaths: ['LICENSE'], + expectedKinds: ['skill'] + }, + { + evidenceId: 'roadrunner-admin', + repository: 'roadrunner-tuff/roadrunner-admin-plugin', + repositoryUrl: 'https://github.com/roadrunner-tuff/roadrunner-admin-plugin', + commit: '8e130c07656c8f9db8bf5431332c9aed60a4b133', + archiveUrl: 'https://codeload.github.com/roadrunner-tuff/roadrunner-admin-plugin/tar.gz/8e130c07656c8f9db8bf5431332c9aed60a4b133', + archiveRoot: 'roadrunner-admin-plugin-8e130c07656c8f9db8bf5431332c9aed60a4b133', + classification: 'DIRECT', + adapterId: 'none', + marketplaceRoot: '.', + plugin: 'roadrunner-admin', + marketplace: 'roadrunner', + license: 'Apache-2.0', + licensePaths: ['LICENSE'], + expectedKinds: ['skill', 'mcp'] + } +]; + +export const PUBLIC_FIXTURES = Object.freeze( + FIXED_FIXTURES.map((fixture) => Object.freeze({ + ...fixture, + licensePaths: Object.freeze([...fixture.licensePaths]), + expectedKinds: Object.freeze([...fixture.expectedKinds]) + })) +); + +const FIXED_FIXTURE_JSON = JSON.stringify(FIXED_FIXTURES); +const SHA256 = /^[a-f0-9]{64}$/; +const COMMIT_SHA = /^[a-f0-9]{40}$/; +const EXPECTED_SOURCE = { + source: 'url', + url: 'https://github.com/Cassette-Editor/oh-my-cassette.git', + ref: 'release' +}; +const EXPECTED_MANIFEST_SHA256 = 'd5c629f3a3b8dd2cdf560963c26b1c5e9dc062045fef13cafca178e7b1bd3d3f'; +const DEFAULT_ARCHIVE_LIMIT = 128 * 1024 * 1024; +const DEFAULT_UNCOMPRESSED_LIMIT = 512 * 1024 * 1024; +const DEFAULT_FETCH_TIMEOUT_MS = 60_000; +const MAX_ARCHIVE_ENTRIES = 100_000; +const PROBE_RECEIPT_LIMIT = 1024 * 1024; +const DOCKER_TIMEOUT_MS = 10_000; +const DOCKER_OUTPUT_LIMIT = 64 * 1024; + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +export function validateFixtureDefinitions(fixtures) { + if (!Array.isArray(fixtures) || JSON.stringify(fixtures) !== FIXED_FIXTURE_JSON) { + throw new Error('Public matrix must use the fixed fixture definitions'); + } + return fixtures; +} + +export function publicFixtureOptionsFromEnvironment(env = process.env, cwd = process.cwd()) { + if ( + env.CODEX_CURRENT_VERSION !== CODEX_VERSIONS[0] || + env.CODEX_PRIOR_VERSION !== CODEX_VERSIONS[1] + ) { + throw new Error( + `Public fixture matrix requires CODEX_CURRENT_VERSION=${CODEX_VERSIONS[0]} and ` + + `CODEX_PRIOR_VERSION=${CODEX_VERSIONS[1]}` + ); + } + const requested = env.CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT; + const outputBoundary = path.resolve(cwd); + const outputRoot = path.resolve(outputBoundary, requested ?? ''); + const relative = path.relative(outputBoundary, outputRoot); + if ( + typeof requested !== 'string' || + requested.trim() === '' || + path.isAbsolute(requested) || + relative === '' || + relative === '..' || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw new Error('CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT must name a relative evidence directory'); + } + return { + architecture: process.arch, + outputBoundary, + outputRoot, + platform: process.platform + }; +} + +function safeArchivePath(value, label, allowRelativeSegments = false) { + if ( + typeof value !== 'string' || + value === '' || + value.includes('\\') || + value.includes('\0') || + value.startsWith('/') || + /^[A-Za-z]:/.test(value) + ) { + throw new Error(`Unsafe archive ${label}`); + } + const withoutTrailingSlash = value.endsWith('/') ? value.slice(0, -1) : value; + const components = withoutTrailingSlash.split('/'); + if (components.some((component) => + component === '' || (!allowRelativeSegments && (component === '.' || component === '..')) + )) { + throw new Error(`Unsafe archive ${label}`); + } + return withoutTrailingSlash; +} + +function isInsideArchiveRoot(value, root) { + return value === root || value.startsWith(`${root}/`); +} + +export function validateArchiveEntries(entries, expectedRoot) { + if (!Array.isArray(entries) || entries.length === 0) { + throw new Error('Archive must contain entries'); + } + if (!COMMIT_SHA.test(expectedRoot.slice(expectedRoot.lastIndexOf('-') + 1))) { + throw new Error('Archive root must end with the immutable commit'); + } + const seen = new Set(); + for (const entry of entries) { + const entryPath = safeArchivePath(entry?.path, 'path'); + if (!isInsideArchiveRoot(entryPath, expectedRoot)) { + throw new Error('Archive entry escaped the fixed root'); + } + if (seen.has(entryPath)) throw new Error(`Duplicate archive path: ${entryPath}`); + seen.add(entryPath); + if (!['file', 'directory', 'symlink', 'hardlink'].includes(entry?.type)) { + throw new Error(`Unsupported archive entry type: ${entry?.type}`); + } + if (entry.type === 'symlink') { + const linkPath = safeArchivePath(entry.linkPath, 'symlink target', true); + const resolved = path.posix.normalize(path.posix.join(path.posix.dirname(entryPath), linkPath)); + if (!isInsideArchiveRoot(resolved, expectedRoot)) { + throw new Error('Archive symlink escaped the fixed root'); + } + } + if (entry.type === 'hardlink') { + const linkPath = safeArchivePath(entry.linkPath, 'hardlink target', true); + const resolved = path.posix.normalize(linkPath); + if (!isInsideArchiveRoot(resolved, expectedRoot)) { + throw new Error('Archive hardlink escaped the fixed root'); + } + } + } + return entries; +} + +export function applyLocalSourceAdapter(input) { + const bytes = Buffer.from(input); + const originalSha256 = sha256(bytes); + let manifest; + try { + manifest = JSON.parse(bytes.toString('utf8')); + } catch { + throw new Error('Immutable adapter precondition failed: manifest is not JSON'); + } + if ( + originalSha256 !== EXPECTED_MANIFEST_SHA256 || + manifest?.plugins?.length !== 1 || + manifest.plugins[0]?.name !== 'oh-my-cassette' || + JSON.stringify(manifest.plugins[0].source) !== JSON.stringify(EXPECTED_SOURCE) + ) { + throw new Error('Immutable adapter precondition failed'); + } + const adapted = structuredClone(manifest); + adapted.plugins[0].source = { source: 'local', path: './' }; + const adaptedBytes = Buffer.from(`${JSON.stringify(adapted, null, 2)}\n`); + return { + bytes: adaptedBytes, + originalSha256, + adaptedSha256: sha256(adaptedBytes) + }; +} + +function assertFixedFixture(fixture) { + const fixed = FIXED_FIXTURES.find(({ repository }) => repository === fixture?.repository); + if (fixed === undefined || JSON.stringify(fixture) !== JSON.stringify(fixed)) { + throw new Error('Archive request must use an exact fixed fixture definition'); + } + return fixture; +} + +export async function fetchPublicArchive(fixture, dependencies = {}) { + assertFixedFixture(fixture); + const fetchImpl = dependencies.fetchImpl ?? globalThis.fetch; + if (typeof fetchImpl !== 'function') throw new Error('Public archive fetch is unavailable'); + const maxBytes = dependencies.maxBytes ?? DEFAULT_ARCHIVE_LIMIT; + const timeoutMs = dependencies.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS; + if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) { + throw new Error('Public archive size limit must be a positive integer'); + } + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new Error('Public archive timeout must be a positive integer'); + } + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + timer.unref?.(); + let response; + try { + response = await fetchImpl(fixture.archiveUrl, { + method: 'GET', + redirect: 'error', + credentials: 'omit', + headers: { + accept: 'application/x-gzip, application/octet-stream', + 'user-agent': 'codex-plugin-check-public-fixture/0.1.0' + }, + signal: controller.signal + }); + if (!response?.ok || response.status !== 200) { + throw new Error(`Public archive fetch failed with HTTP ${response?.status ?? 'unknown'}`); + } + if (response.url !== fixture.archiveUrl) { + throw new Error('Public archive response URL did not match the exact codeload identity'); + } + const contentLengthValue = response.headers?.get?.('content-length'); + if (contentLengthValue !== null && contentLengthValue !== undefined) { + if (!/^\d+$/.test(contentLengthValue)) { + throw new Error('Public archive returned an invalid Content-Length'); + } + const contentLength = Number(contentLengthValue); + if (!Number.isSafeInteger(contentLength) || contentLength > maxBytes) { + throw new Error(`Public archive size exceeds ${maxBytes} bytes`); + } + } + const reader = response.body?.getReader?.(); + if (reader === undefined) throw new Error('Public archive response body is not streamable'); + const chunks = []; + let observedBytes = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = Buffer.from(value); + observedBytes += chunk.length; + if (observedBytes > maxBytes) { + await reader.cancel().catch(() => {}); + throw new Error(`Public archive size exceeds ${maxBytes} bytes`); + } + chunks.push(chunk); + } + if (observedBytes === 0) throw new Error('Public archive response was empty'); + const bytes = Buffer.concat(chunks, observedBytes); + return { bytes, archiveSha256: sha256(bytes) }; + } catch (cause) { + if (controller.signal.aborted) { + throw new Error(`Public archive fetch timed out after ${timeoutMs}ms`, { cause }); + } + throw cause; + } finally { + clearTimeout(timer); + } +} + +function tarString(field) { + const zero = field.indexOf(0); + const value = field.subarray(0, zero === -1 ? field.length : zero).toString('utf8'); + if (value.includes('\uFFFD')) throw new Error('Archive header contained invalid UTF-8'); + return value; +} + +function tarNumber(field, label) { + if ((field[0] & 0x80) !== 0) { + throw new Error(`Archive ${label} uses unsupported base-256 encoding`); + } + const value = tarString(field).trim(); + if (value === '') return 0; + if (!/^[0-7]+$/.test(value)) throw new Error(`Archive ${label} is not octal`); + const number = Number.parseInt(value, 8); + if (!Number.isSafeInteger(number) || number < 0) { + throw new Error(`Archive ${label} is outside the safe integer range`); + } + return number; +} + +function verifyTarChecksum(header) { + const expected = tarNumber(header.subarray(148, 156), 'checksum'); + const copy = Buffer.from(header); + copy.fill(0x20, 148, 156); + const observed = copy.reduce((sum, byte) => sum + byte, 0); + if (observed !== expected) throw new Error('Archive header checksum did not match'); +} + +function parsePaxRecords(data) { + const records = {}; + let offset = 0; + while (offset < data.length) { + const space = data.indexOf(0x20, offset); + if (space === -1) throw new Error('Archive PAX record is missing a length separator'); + const lengthText = data.subarray(offset, space).toString('ascii'); + if (!/^[1-9]\d*$/.test(lengthText)) throw new Error('Archive PAX record has invalid length'); + const length = Number(lengthText); + if (!Number.isSafeInteger(length) || offset + length > data.length) { + throw new Error('Archive PAX record exceeds its payload'); + } + const record = data.subarray(space + 1, offset + length); + if (record.at(-1) !== 0x0a) throw new Error('Archive PAX record is not newline terminated'); + const content = record.subarray(0, -1).toString('utf8'); + if (content.includes('\uFFFD')) throw new Error('Archive PAX record contained invalid UTF-8'); + const equals = content.indexOf('='); + if (equals <= 0) throw new Error('Archive PAX record is missing a key'); + const key = content.slice(0, equals); + if (Object.hasOwn(records, key)) throw new Error(`Archive PAX key is duplicated: ${key}`); + records[key] = content.slice(equals + 1); + offset += length; + } + return records; +} + +function paddedTarSize(size) { + return Math.ceil(size / 512) * 512; +} + +export function parseTarGzipArchive(archive, options) { + const compressed = Buffer.from(archive); + const maxUncompressedBytes = options.maxUncompressedBytes ?? DEFAULT_UNCOMPRESSED_LIMIT; + if (!Number.isSafeInteger(maxUncompressedBytes) || maxUncompressedBytes <= 0) { + throw new Error('Archive uncompressed size limit must be a positive integer'); + } + let tar; + try { + tar = gunzipSync(compressed, { maxOutputLength: maxUncompressedBytes }); + } catch (cause) { + throw new Error('Archive gzip payload is invalid or exceeds its size limit', { cause }); + } + if (tar.length === 0 || tar.length > maxUncompressedBytes) { + throw new Error('Archive uncompressed payload is empty or exceeds its size limit'); + } + const entries = []; + let offset = 0; + let nextPax = null; + let pendingLongName = null; + let pendingLongLink = null; + let sawTerminator = false; + while (offset + 512 <= tar.length) { + const header = tar.subarray(offset, offset + 512); + offset += 512; + if (header.every((byte) => byte === 0)) { + sawTerminator = true; + break; + } + verifyTarChecksum(header); + const headerSize = tarNumber(header.subarray(124, 136), 'size'); + if (offset + paddedTarSize(headerSize) > tar.length) { + throw new Error('Archive entry payload exceeds the tar stream'); + } + const headerData = Buffer.from(tar.subarray(offset, offset + headerSize)); + offset += paddedTarSize(headerSize); + const typeFlag = header[156] === 0 ? '0' : String.fromCharCode(header[156]); + if (typeFlag === 'g') { + const globalPax = parsePaxRecords(headerData); + if (['path', 'linkpath', 'size'].some((key) => Object.hasOwn(globalPax, key))) { + throw new Error('Archive global PAX metadata may not override paths or sizes'); + } + continue; + } + if (typeFlag === 'x') { + if (nextPax !== null) throw new Error('Archive contains stacked local PAX metadata'); + nextPax = parsePaxRecords(headerData); + continue; + } + if (typeFlag === 'L' || typeFlag === 'K') { + const value = tarString(headerData); + if (value === '') throw new Error('Archive GNU long path metadata was empty'); + if (typeFlag === 'L') pendingLongName = value; + else pendingLongLink = value; + continue; + } + + const prefix = tarString(header.subarray(345, 500)); + const name = tarString(header.subarray(0, 100)); + const headerPath = prefix === '' ? name : `${prefix}/${name}`; + const entryPath = nextPax?.path ?? pendingLongName ?? headerPath; + const linkPath = nextPax?.linkpath ?? pendingLongLink ?? tarString(header.subarray(157, 257)); + const effectiveSizeText = nextPax?.size; + let effectiveSize = headerSize; + if (effectiveSizeText !== undefined) { + if (!/^\d+$/.test(effectiveSizeText)) throw new Error('Archive PAX size is invalid'); + effectiveSize = Number(effectiveSizeText); + if (!Number.isSafeInteger(effectiveSize) || effectiveSize !== headerSize) { + throw new Error('Archive PAX size disagrees with the bounded header payload'); + } + } + nextPax = null; + pendingLongName = null; + pendingLongLink = null; + const type = typeFlag === '0' + ? 'file' + : typeFlag === '5' + ? 'directory' + : typeFlag === '2' + ? 'symlink' + : typeFlag === '1' + ? 'hardlink' + : null; + if (type === null) throw new Error(`Unsupported archive entry type: ${typeFlag}`); + if (type !== 'file' && effectiveSize !== 0) { + throw new Error(`Archive ${type} entry unexpectedly contained data`); + } + entries.push({ + path: entryPath, + type, + linkPath: type === 'symlink' || type === 'hardlink' ? linkPath : undefined, + mode: tarNumber(header.subarray(100, 108), 'mode'), + data: type === 'file' ? headerData : undefined + }); + if (entries.length > MAX_ARCHIVE_ENTRIES) { + throw new Error(`Archive exceeds ${MAX_ARCHIVE_ENTRIES} entries`); + } + } + if (!sawTerminator) throw new Error('Archive tar stream is missing its zero terminator'); + if (nextPax !== null || pendingLongName !== null || pendingLongLink !== null) { + throw new Error('Archive ended with unapplied path metadata'); + } + return validateArchiveEntries(entries, options.expectedRoot); +} + +function archiveRelativePath(entryPath, expectedRoot) { + const normalized = entryPath.endsWith('/') ? entryPath.slice(0, -1) : entryPath; + return normalized === expectedRoot ? '' : normalized.slice(expectedRoot.length + 1); +} + +async function makeSafeDirectory(root, relative) { + if (relative === '') return; + let current = root; + for (const component of relative.split('/')) { + current = path.join(current, component); + try { + const metadata = await lstat(current); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + throw new Error(`Archive directory path is not an owned directory: ${relative}`); + } + } catch (cause) { + if (cause?.code !== 'ENOENT') throw cause; + await mkdir(current, { mode: 0o700 }); + } + } +} + +export async function extractArchiveEntries(entries, destination, expectedRoot) { + validateArchiveEntries(entries, expectedRoot); + const destinationRoot = await realpath(path.resolve(destination)); + const checkoutRoot = path.join(destinationRoot, expectedRoot); + try { + await lstat(checkoutRoot); + throw new Error('Archive checkout root must not already exist'); + } catch (cause) { + if (cause?.code !== 'ENOENT') throw cause; + } + await mkdir(checkoutRoot, { mode: 0o700 }); + + const directories = entries + .filter(({ type }) => type === 'directory') + .map(({ path: entryPath }) => archiveRelativePath(entryPath, expectedRoot)) + .filter((relative) => relative !== '') + .sort((left, right) => left.split('/').length - right.split('/').length); + for (const relative of directories) await makeSafeDirectory(checkoutRoot, relative); + + for (const entry of entries.filter(({ type }) => type === 'file')) { + const relative = archiveRelativePath(entry.path, expectedRoot); + if (relative === '') throw new Error('Archive root may not be a file'); + await makeSafeDirectory(checkoutRoot, path.posix.dirname(relative) === '.' + ? '' + : path.posix.dirname(relative)); + await writeFile(path.join(checkoutRoot, ...relative.split('/')), entry.data, { + flag: 'wx', + mode: 0o600 + }); + } + + for (const entry of entries.filter(({ type }) => type === 'hardlink')) { + const relative = archiveRelativePath(entry.path, expectedRoot); + const targetRelative = archiveRelativePath(entry.linkPath, expectedRoot); + if (relative === '' || targetRelative === '') throw new Error('Archive hardlink path is invalid'); + await makeSafeDirectory(checkoutRoot, path.posix.dirname(relative) === '.' + ? '' + : path.posix.dirname(relative)); + const target = path.join(checkoutRoot, ...targetRelative.split('/')); + const targetMetadata = await lstat(target); + if (!targetMetadata.isFile() || targetMetadata.isSymbolicLink()) { + throw new Error('Archive hardlink target must be an extracted regular file'); + } + await link(target, path.join(checkoutRoot, ...relative.split('/'))); + } + + for (const entry of entries.filter(({ type }) => type === 'symlink')) { + const relative = archiveRelativePath(entry.path, expectedRoot); + if (relative === '') throw new Error('Archive root may not be a symlink'); + await makeSafeDirectory(checkoutRoot, path.posix.dirname(relative) === '.' + ? '' + : path.posix.dirname(relative)); + await symlink(entry.linkPath, path.join(checkoutRoot, ...relative.split('/'))); + } + await auditExtractedSymlinks(checkoutRoot); + return await realpath(checkoutRoot); +} + +export async function auditExtractedSymlinks(checkoutRoot) { + const canonicalRoot = await realpath(checkoutRoot); + async function visit(directory, relativeDirectory = '') { + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries) { + const relative = relativeDirectory === '' + ? entry.name + : `${relativeDirectory}/${entry.name}`; + const absolute = path.join(directory, entry.name); + if (entry.isSymbolicLink()) { + const target = await readlink(absolute); + if (path.isAbsolute(target) || target.includes('\0') || target.includes('\\')) { + throw new Error(`Extracted symlink escaped checkout: ${relative}`); + } + const resolved = path.resolve(path.dirname(absolute), target); + if (!pathIsWithin(resolved, canonicalRoot)) { + throw new Error(`Extracted symlink escaped checkout: ${relative}`); + } + } else if (entry.isDirectory()) { + await visit(absolute, relative); + } else if (!entry.isFile()) { + throw new Error(`Extracted archive contains unsupported filesystem entry: ${relative}`); + } + } + } + await visit(canonicalRoot); +} + +async function hashDirectory(root) { + const hash = createHash('sha256'); + async function visit(directory, relativeDirectory = '') { + const entries = await readdir(directory, { withFileTypes: true }); + entries.sort((left, right) => left.name.localeCompare(right.name, 'en')); + for (const entry of entries) { + const relative = relativeDirectory === '' + ? entry.name + : `${relativeDirectory}/${entry.name}`; + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) { + hash.update(`directory\0${relative}\0`); + await visit(absolute, relative); + } else if (entry.isFile()) { + hash.update(`file\0${relative}\0`); + hash.update(await readFile(absolute)); + hash.update('\0'); + } else if (entry.isSymbolicLink()) { + hash.update(`symlink\0${relative}\0${await readlink(absolute)}\0`); + } else { + throw new Error(`Cannot hash unsupported fixture entry: ${relative}`); + } + } + } + await visit(root); + return hash.digest('hex'); +} + +async function assertRegularFixtureFile(checkoutRoot, relative, label) { + const absolute = path.resolve(checkoutRoot, ...relative.split('/')); + if (!pathIsWithin(absolute, checkoutRoot) || absolute === checkoutRoot) { + throw new Error(`${label} path escaped the fixture checkout`); + } + const metadata = await lstat(absolute); + if (metadata.isSymbolicLink() || !metadata.isFile()) { + throw new Error(`${label} must be a regular file, not a symlink`); + } + return absolute; +} + +export async function preparePublicFixture({ fixture, temporaryRoot }, dependencies = {}) { + assertFixedFixture(fixture); + const canonicalTemporaryRoot = await realpath(temporaryRoot); + const fixtureStateRoot = path.join(canonicalTemporaryRoot, `fixture-${fixture.evidenceId}`); + await mkdir(fixtureStateRoot, { mode: 0o700 }); + const fetchArchive = dependencies.fetchArchive ?? fetchPublicArchive; + const fetched = await fetchArchive(fixture); + if (!Buffer.isBuffer(fetched?.bytes)) throw new Error('Public archive fetch returned no bytes'); + const archiveSha256 = sha256(fetched.bytes); + if (fetched.archiveSha256 !== archiveSha256) { + throw new Error('Public archive SHA256 did not match its downloaded bytes'); + } + const entries = parseTarGzipArchive(fetched.bytes, { + expectedRoot: fixture.archiveRoot, + maxUncompressedBytes: dependencies.maxUncompressedBytes ?? DEFAULT_UNCOMPRESSED_LIMIT + }); + const checkoutRoot = await extractArchiveEntries( + entries, + fixtureStateRoot, + fixture.archiveRoot + ); + for (const licensePath of fixture.licensePaths) { + const license = await assertRegularFixtureFile(checkoutRoot, licensePath, 'License or notice'); + if ((await readFile(license)).length === 0) { + throw new Error(`License or notice file was empty: ${licensePath}`); + } + } + const checkoutSha256 = await hashDirectory(checkoutRoot); + let adapter = null; + if (fixture.adapterId === 'local-source-v1') { + const adapterPath = await assertRegularFixtureFile( + checkoutRoot, + fixture.adapterPath, + 'Adapter manifest' + ); + const adapted = applyLocalSourceAdapter(await readFile(adapterPath)); + const temporaryAdapterPath = path.join(path.dirname(adapterPath), '.marketplace.adapter.tmp'); + await writeFile(temporaryAdapterPath, adapted.bytes, { flag: 'wx', mode: 0o600 }); + await rename(temporaryAdapterPath, adapterPath); + adapter = { + id: fixture.adapterId, + originalSha256: adapted.originalSha256, + adaptedSha256: adapted.adaptedSha256 + }; + } + const marketplaceRequest = path.resolve(checkoutRoot, ...fixture.marketplaceRoot.split('/')); + if (!pathIsWithin(marketplaceRequest, checkoutRoot)) { + throw new Error('Marketplace root escaped the fixture checkout'); + } + const marketplaceRoot = await realpath(marketplaceRequest); + const marketplaceMetadata = await lstat(marketplaceRoot); + if (!marketplaceMetadata.isDirectory() || !pathIsWithin(marketplaceRoot, checkoutRoot)) { + throw new Error('Marketplace root must be an owned fixture directory'); + } + await auditExtractedSymlinks(checkoutRoot); + return { + archiveSha256, + checkoutSha256, + marketplaceSha256: await hashDirectory(marketplaceRoot), + marketplaceRoot, + adapter + }; +} + +function allStringValues(value, values = []) { + if (typeof value === 'string') { + values.push(value); + } else if (Array.isArray(value)) { + for (const item of value) allStringValues(item, values); + } else if (value !== null && typeof value === 'object') { + for (const item of Object.values(value)) allStringValues(item, values); + } + return values; +} + +export function validatePublicReceipt(receipt, options) { + const { architecture, fixture, version } = options; + const receiptCode = validateReceipt(receipt, { + codexVersion: version, + plugin: fixture.plugin, + sourceRoot: '/workspace', + platform: `linux-${architecture}`, + isolation: { mode: 'strict', network: 'denied', hostState: 'denied' } + }); + const expectedExitCode = options.expectedExitCode ?? 0; + const expectedStatus = expectedExitCode === 0 + ? 'PASS' + : expectedExitCode === 1 + ? 'FAIL' + : null; + if (expectedStatus === null || receiptCode !== expectedExitCode || receipt.status !== expectedStatus) { + throw new Error( + `Public fixture ${fixture.repository} receipt did not match expected ${expectedStatus ?? 'exit'}` + ); + } + if (receipt.plugin.marketplace !== fixture.marketplace) { + throw new Error(`Public fixture ${fixture.repository} returned the wrong marketplace identity`); + } + const expectedKinds = new Set(fixture.expectedKinds); + const observedKinds = new Set(); + const capabilityKeys = new Set(); + for (const capability of receipt.capabilities) { + if (!expectedKinds.has(capability.kind)) { + throw new Error(`Public fixture ${fixture.repository} returned an unexpected capability kind`); + } + const identity = `${capability.kind}\0${capability.key}`; + if (capabilityKeys.has(identity)) { + throw new Error(`Public fixture ${fixture.repository} returned a duplicate capability`); + } + capabilityKeys.add(identity); + observedKinds.add(capability.kind); + if ( + capability.kind === 'skill' && + !['DISCOVERED_EFFECTIVE', ...(expectedExitCode === 1 ? ['MISSING'] : [])] + .includes(capability.status) + ) { + throw new Error(`Public fixture ${fixture.repository} skill was not effective`); + } + if ( + capability.kind === 'hook' && + ![ + 'DISCOVERED_UNTRUSTED', + 'DISCOVERED_EFFECTIVE', + ...(expectedExitCode === 1 ? ['MISSING'] : []) + ].includes(capability.status) + ) { + throw new Error(`Public fixture ${fixture.repository} hook trust state was not observed`); + } + if ( + capability.kind === 'mcp' && + !['DECLARED_ONLY', ...(expectedExitCode === 1 ? ['MISSING'] : [])] + .includes(capability.status) + ) { + throw new Error(`Public fixture ${fixture.repository} MCP capability made a runtime claim`); + } + } + for (const kind of expectedKinds) { + if (!observedKinds.has(kind)) { + throw new Error(`Public fixture ${fixture.repository} is missing capability kind ${kind}`); + } + } + const markers = options.personalMarkers ?? []; + const values = allStringValues(receipt); + for (const marker of markers) { + if (typeof marker === 'string' && marker !== '' && values.some((value) => value.includes(marker))) { + throw new Error(`Public fixture ${fixture.repository} receipt leaked a personal host path`); + } + } + for (const value of values) { + if ( + value !== '/workspace' && + (/^(?:\/Users\/|\/home\/)/.test(value) || /^[A-Za-z]:[\\/]/.test(value)) + ) { + throw new Error(`Public fixture ${fixture.repository} receipt leaked an absolute host path`); + } + } + return receipt; +} + +export function validateRelativeEvidencePath(value) { + if ( + typeof value !== 'string' || + value === '' || + value === '.' || + value === '..' || + path.isAbsolute(value) || + value.includes('/') || + value.includes('\\') || + value.includes('\0') + ) { + throw new Error('Evidence output must use a relative evidence path filename'); + } + return value; +} + +function pathIsWithin(candidate, root) { + const relative = path.relative(root, candidate); + return relative === '' || ( + relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); +} + +async function createOwnedOutputRoot(requestedRoot, requestedBoundary) { + const boundary = await realpath(path.resolve(requestedBoundary)); + const outputRequest = path.resolve(requestedRoot); + const relative = path.relative(boundary, outputRequest); + if ( + relative === '' || + relative === '..' || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw new Error('Evidence output root must be inside its checkout boundary'); + } + let current = boundary; + const components = relative.split(path.sep); + for (let index = 0; index < components.length; index += 1) { + current = path.join(current, components[index]); + const isOutputRoot = index === components.length - 1; + try { + const metadata = await lstat(current); + if (metadata.isSymbolicLink()) { + throw new Error(`Evidence output parent must not be a symlink: ${current}`); + } + if (!metadata.isDirectory()) { + throw new Error(`Evidence output parent must be a directory: ${current}`); + } + if (isOutputRoot) { + throw new Error('Evidence output root must not already exist'); + } + } catch (cause) { + if (cause?.code !== 'ENOENT') throw cause; + await mkdir(current, { mode: 0o700 }); + } + } + const outputRoot = await realpath(outputRequest); + if (!pathIsWithin(outputRoot, boundary)) { + throw new Error('Evidence output root escaped its checkout boundary'); + } + return { boundary, outputRoot }; +} + +async function removeOwnedOutputRoot(outputRoot, boundary) { + let canonical; + try { + canonical = await realpath(outputRoot); + } catch (cause) { + if (cause?.code === 'ENOENT') return; + throw cause; + } + if (canonical !== outputRoot || !pathIsWithin(canonical, boundary) || canonical === boundary) { + throw new Error('Refusing to clean an unowned evidence output root'); + } + await rm(canonical, { recursive: true, force: true }); +} + +export async function writeAtomicEvidenceJson(binding, filename, value, dependencies = {}) { + validateRelativeEvidencePath(filename); + await assertOwnedOutputRoot(binding.outputRoot, binding.boundary); + const openFile = dependencies.open ?? open; + const createLink = dependencies.link ?? link; + const removeLink = dependencies.unlink ?? unlink; + const identifier = randomUUID().replace(/[^A-Za-z0-9_-]/g, ''); + const temporary = path.join(binding.outputRoot, `.${filename}.${identifier}.tmp`); + const destination = path.join(binding.outputRoot, filename); + const flags = fsConstants.O_WRONLY | + fsConstants.O_CREAT | + fsConstants.O_EXCL | + (fsConstants.O_NOFOLLOW ?? 0); + let handle; + let temporaryExists = false; + try { + handle = await openFile(temporary, flags, 0o600); + temporaryExists = true; + await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8'); + await handle.sync(); + await handle.close(); + handle = undefined; + await assertOwnedOutputRoot(binding.outputRoot, binding.boundary); + try { + await createLink(temporary, destination); + } catch (cause) { + if (cause?.code === 'EEXIST') { + throw new Error(`Fresh atomic evidence destination already exists: ${filename}`, { + cause + }); + } + throw cause; + } + await removeLink(temporary); + temporaryExists = false; + } catch (cause) { + if (handle !== undefined) { + try { + await handle.close(); + } catch { + // Preserve the primary evidence-delivery error. + } + } + if (temporaryExists) { + try { + await removeLink(temporary); + } catch { + // Preserve the primary evidence-delivery error. + } + } + throw cause; + } +} + +async function assertOwnedOutputRoot(outputRoot, boundary) { + const metadata = await lstat(outputRoot); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + throw new Error('Evidence output root must remain an owned directory, not a symlink'); + } + const canonical = await realpath(outputRoot); + if (canonical !== outputRoot || !pathIsWithin(canonical, boundary)) { + throw new Error('Evidence output root escaped its checkout boundary'); + } +} + +function boundedIoWriter(chunks, state) { + return { + write(value) { + const chunk = Buffer.from(String(value)); + state.bytes += chunk.length; + if (state.bytes > DOCKER_OUTPUT_LIMIT) { + throw new Error(`Tool output exceeded ${DOCKER_OUTPUT_LIMIT} bytes`); + } + chunks.push(chunk); + } + }; +} + +export async function probePublicFixtureCell(options, dependencies = {}) { + assertFixedFixture(options.fixture); + if (!CODEX_VERSIONS.includes(options.version)) { + throw new Error('Public fixture probe requires an exact fixed Codex version'); + } + const temporaryRoot = await realpath(options.temporaryRoot); + const marketplaceRoot = await realpath(options.marketplaceRoot); + if (!pathIsWithin(marketplaceRoot, temporaryRoot)) { + throw new Error('Public fixture probe marketplace escaped owned temporary state'); + } + const stagedDirectory = path.join(temporaryRoot, 'staged-receipts'); + try { + const metadata = await lstat(stagedDirectory); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + throw new Error('Staged receipt parent must be an owned directory, not a symlink'); + } + } catch (cause) { + if (cause?.code !== 'ENOENT') throw cause; + await mkdir(stagedDirectory, { mode: 0o700 }); + } + const stagedPath = path.join( + stagedDirectory, + `${options.fixture.evidenceId}--codex-${options.version}.json` + ); + try { + await lstat(stagedPath); + throw new Error('Public fixture staged receipt must be fresh and must not already exist'); + } catch (cause) { + if (cause?.code !== 'ENOENT') throw cause; + } + + const stdout = []; + const stderr = []; + const stdoutState = { bytes: 0 }; + const stderrState = { bytes: 0 }; + const cliMain = dependencies.cliMain ?? cliMainReal; + let invoked = false; + try { + invoked = true; + const code = await cliMain([ + '--marketplace-root', marketplaceRoot, + '--plugin', options.fixture.plugin, + '--codex-version', options.version, + '--cwd', marketplaceRoot, + '--output', stagedPath, + '--isolation', 'strict', + '--quiet' + ], { + stdout: boundedIoWriter(stdout, stdoutState), + stderr: boundedIoWriter(stderr, stderrState) + }); + if (code !== 0 && code !== 1) { + const detail = Buffer.concat(stderr).toString('utf8').trim(); + throw new Error( + `Public fixture CLI returned unexpected code ${code}${detail ? `: ${detail}` : ''}` + ); + } + const metadata = await lstat(stagedPath); + if (metadata.isSymbolicLink() || !metadata.isFile() || metadata.size > PROBE_RECEIPT_LIMIT) { + throw new Error('Public fixture staged receipt was not a bounded regular file'); + } + let receipt; + try { + receipt = JSON.parse(await readFile(stagedPath, 'utf8')); + } catch (cause) { + throw new Error('Public fixture staged receipt was not valid JSON', { cause }); + } + validatePublicReceipt(receipt, { + architecture: options.architecture, + expectedExitCode: code, + fixture: options.fixture, + personalMarkers: options.personalMarkers, + version: options.version + }); + return { code, receipt }; + } finally { + if (invoked) await rm(stagedPath, { force: true }); + } +} + +function inspectDockerReal() { + return new Promise((resolve, reject) => { + const child = spawn('docker', ['version', '--format', '{{.Server.Version}}'], { + shell: false, + stdio: ['ignore', 'pipe', 'pipe'] + }); + const stdout = []; + const stderr = []; + let outputBytes = 0; + let forcedError; + let settled = false; + const terminate = (cause) => { + forcedError ??= cause; + child.kill('SIGTERM'); + }; + for (const [stream, chunks] of [[child.stdout, stdout], [child.stderr, stderr]]) { + stream.on('data', (chunk) => { + outputBytes += chunk.length; + if (outputBytes > DOCKER_OUTPUT_LIMIT) { + terminate(new Error(`Docker output exceeded ${DOCKER_OUTPUT_LIMIT} bytes`)); + return; + } + chunks.push(Buffer.from(chunk)); + }); + } + const timer = setTimeout( + () => terminate(new Error(`Docker inspection timed out after ${DOCKER_TIMEOUT_MS}ms`)), + DOCKER_TIMEOUT_MS + ); + timer.unref?.(); + child.once('error', (cause) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(cause); + }); + child.once('close', (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (forcedError) { + reject(forcedError); + return; + } + resolve({ + code, + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8') + }); + }); + }); +} + +export async function assertPublicFixtureRuntime(options, dependencies = {}) { + const platform = options.platform ?? process.platform; + if (platform !== 'linux') { + throw new Error(`Strict public fixture matrix requires Linux, observed ${platform}`); + } + const inspectDocker = dependencies.inspectDocker ?? inspectDockerReal; + let result; + try { + result = await inspectDocker(); + } catch (cause) { + throw new Error(`Strict public fixture matrix requires Docker: ${cause.message}`, { cause }); + } + if (result?.code !== 0 || String(result?.stdout ?? '').trim() === '') { + const detail = String(result?.stderr ?? '').trim(); + throw new Error(`Strict public fixture matrix requires Docker${detail ? `: ${detail}` : ''}`); + } +} + +function sanitizeReceipt(receipt) { + return { + schemaVersion: receipt.schemaVersion, + status: receipt.status, + codexVersion: receipt.codexVersion, + platform: receipt.platform, + plugin: { + name: receipt.plugin.name, + marketplace: receipt.plugin.marketplace, + sourceRoot: receipt.plugin.sourceRoot + }, + capabilities: receipt.capabilities.map(({ kind, key, source, status }) => ({ + kind, + key, + source, + status + })), + isolation: { + mode: receipt.isolation.mode, + network: receipt.isolation.network, + hostState: receipt.isolation.hostState + } + }; +} + +function validatePreparedFixture(prepared, fixture, temporaryRoot) { + for (const field of ['archiveSha256', 'checkoutSha256', 'marketplaceSha256']) { + if (!SHA256.test(prepared?.[field] ?? '')) { + throw new Error(`Prepared public fixture is missing ${field}`); + } + } + if (fixture.adapterId === 'none') { + if (prepared.adapter !== null) throw new Error('DIRECT fixture must not record an adapter'); + } else if ( + prepared.adapter?.id !== 'local-source-v1' || + !SHA256.test(prepared.adapter.originalSha256 ?? '') || + !SHA256.test(prepared.adapter.adaptedSha256 ?? '') + ) { + throw new Error('STATIC_ADAPTER fixture must record exact adapter hashes'); + } + if (!pathIsWithin(prepared.marketplaceRoot, temporaryRoot)) { + throw new Error('Prepared marketplace root escaped owned temporary state'); + } + return prepared; +} + +function summaryFixture(fixture, prepared, receipts) { + return { + evidenceId: fixture.evidenceId, + repository: fixture.repository, + repositoryUrl: fixture.repositoryUrl, + commit: fixture.commit, + classification: fixture.classification, + adapterId: fixture.adapterId, + marketplaceRoot: fixture.marketplaceRoot, + plugin: fixture.plugin, + marketplace: fixture.marketplace, + license: fixture.license, + licensePaths: [...fixture.licensePaths], + expectedKinds: [...fixture.expectedKinds], + archiveUrl: fixture.archiveUrl, + archiveSha256: prepared.archiveSha256, + checkoutSha256: prepared.checkoutSha256, + marketplaceSha256: prepared.marketplaceSha256, + adapter: prepared.adapter, + receipts + }; +} + +export async function runPublicFixtureMatrix(options, dependencies = {}) { + const fixtures = validateFixtureDefinitions(options.fixtures ?? PUBLIC_FIXTURES); + const versions = options.versions ?? CODEX_VERSIONS; + if (JSON.stringify(versions) !== JSON.stringify(CODEX_VERSIONS)) { + throw new Error('Public fixture matrix requires the fixed Codex versions'); + } + const assertRuntime = dependencies.assertRuntime ?? assertPublicFixtureRuntime; + const makeTemporaryRoot = dependencies.makeTemporaryRoot ?? makeTemporaryRootReal; + const prepareFixture = dependencies.prepareFixture ?? preparePublicFixture; + const probeCell = dependencies.probeCell ?? probePublicFixtureCell; + const removeTemporaryRoot = dependencies.removeTemporaryRoot ?? removeTemporaryRootReal; + + let ownedOutput; + let temporaryRoot; + let primaryError; + try { + await assertRuntime({ platform: options.platform }); + ownedOutput = await createOwnedOutputRoot(options.outputRoot, options.outputBoundary); + temporaryRoot = await realpath(await makeTemporaryRoot()); + const fixturesSummary = []; + let compatibleCells = 0; + let incompatibleCells = 0; + for (const fixture of fixtures) { + const prepared = validatePreparedFixture( + await prepareFixture({ fixture, temporaryRoot }), + fixture, + temporaryRoot + ); + const receipts = []; + for (const version of versions) { + const evidencePath = `${fixture.evidenceId}--codex-${version}.json`; + const staged = await probeCell({ + architecture: options.architecture, + fixture, + marketplaceRoot: prepared.marketplaceRoot, + personalMarkers: options.personalMarkers, + temporaryRoot, + version + }); + if (staged?.code !== 0 && staged?.code !== 1) { + throw new Error( + `Unexpected public fixture tool exit code ${staged?.code}${ + staged?.stderr ? `: ${String(staged.stderr).trim()}` : '' + }` + ); + } + const receipt = sanitizeReceipt(validatePublicReceipt(staged.receipt, { + architecture: options.architecture, + expectedExitCode: staged.code, + fixture, + personalMarkers: options.personalMarkers, + version + })); + await assertOwnedOutputRoot(ownedOutput.outputRoot, ownedOutput.boundary); + await writeAtomicEvidenceJson(ownedOutput, evidencePath, receipt); + const outcome = staged.code === 0 ? 'PASS' : 'VERSION_OR_API_INCOMPATIBLE'; + if (outcome === 'PASS') compatibleCells += 1; + else incompatibleCells += 1; + receipts.push({ version, receiptPath: evidencePath, outcome }); + } + fixturesSummary.push(summaryFixture(fixture, prepared, receipts)); + } + await removeTemporaryRoot(temporaryRoot); + temporaryRoot = undefined; + const summary = { + schemaVersion: '0.1.0', + status: incompatibleCells === 0 ? 'PASS' : 'HOLD', + outputRoot: '.', + platform: `linux-${options.architecture}`, + versions: [...versions], + gate: { + observed: fixtures.length, + required: 10, + cells: fixtures.length * versions.length, + compatibleCells, + incompatibleCells + }, + fixtures: fixturesSummary + }; + await assertOwnedOutputRoot(ownedOutput.outputRoot, ownedOutput.boundary); + await writeAtomicEvidenceJson(ownedOutput, 'public-fixture-summary.json', summary); + return summary; + } catch (cause) { + primaryError = cause; + } + + if (temporaryRoot !== undefined) { + try { + await removeTemporaryRoot(temporaryRoot); + } catch (cause) { + primaryError = new AggregateError([primaryError, cause], 'Public fixture cleanup failed'); + } + } + if (ownedOutput !== undefined) { + try { + await removeOwnedOutputRoot(ownedOutput.outputRoot, ownedOutput.boundary); + } catch (cause) { + primaryError = new AggregateError([primaryError, cause], 'Evidence cleanup failed'); + } + } + throw primaryError; +} + +async function makeTemporaryRootReal() { + const temporaryDirectory = await realpath(os.tmpdir()); + return await realpath(await mkdtemp(path.join( + temporaryDirectory, + 'codex-plugin-public-fixtures-' + ))); +} + +async function removeTemporaryRootReal(root) { + const temporaryDirectory = await realpath(os.tmpdir()); + const canonical = await realpath(root); + if ( + path.dirname(canonical) !== temporaryDirectory || + !path.basename(canonical).startsWith('codex-plugin-public-fixtures-') + ) { + throw new Error(`Refusing to clean unowned public fixture state: ${canonical}`); + } + await rm(canonical, { recursive: true, force: true }); +} + +export async function publicFixtureMain(io = {}, dependencies = {}) { + try { + const options = publicFixtureOptionsFromEnvironment( + io.env ?? process.env, + io.cwd ?? process.cwd() + ); + const runMatrix = dependencies.runMatrix ?? runPublicFixtureMatrix; + const summary = await runMatrix(options); + if (summary?.status === 'PASS') return 0; + if (summary?.status === 'HOLD') return 1; + throw new Error('Public fixture matrix returned an invalid summary status'); + } catch (cause) { + const stderr = io.stderr ?? process.stderr; + stderr?.write?.(`Error: ${cause instanceof Error ? cause.message : String(cause)}\n`); + return 2; + } +} + +function isEntrypoint() { + if (process.argv[1] === undefined) return false; + try { + return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); + } catch { + return false; + } +} + +if (isEntrypoint()) { + process.exitCode = await publicFixtureMain(); +} diff --git a/test/public-fixtures.test.mjs b/test/public-fixtures.test.mjs new file mode 100644 index 0000000..c0cc2fe --- /dev/null +++ b/test/public-fixtures.test.mjs @@ -0,0 +1,1095 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { + access, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + symlink, + writeFile +} from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { gzipSync } from 'node:zlib'; + +import { + applyLocalSourceAdapter, + assertPublicFixtureRuntime, + auditExtractedSymlinks, + CODEX_VERSIONS, + extractArchiveEntries, + fetchPublicArchive, + parseTarGzipArchive, + preparePublicFixture, + probePublicFixtureCell, + publicFixtureMain, + PUBLIC_FIXTURES, + publicFixtureOptionsFromEnvironment, + runPublicFixtureMatrix, + validateArchiveEntries, + validateFixtureDefinitions, + validatePublicReceipt, + validateRelativeEvidencePath, + writeAtomicEvidenceJson +} from '../scripts/falsify-public-fixtures.mjs'; + +const FIXED_IDENTITIES = [ + ['bitrouter/bitrouter', '678384888b73fc290ce4ce503a8a7f2a5cbf6da8', 'DIRECT', 'none'], + ['Cassette-Editor/oh-my-cassette', 'cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32', 'STATIC_ADAPTER', 'local-source-v1'], + ['mostlyharmless-ai/watercooler', 'a5efa89df02e7796e20881fef4847f129d84d367', 'DIRECT', 'none'], + ['commercetools/commercetools-ai-plugins', '440d6bd56eb2969b6a0dd41e3fcff286def0787c', 'DIRECT', 'none'], + ['agentis-tools/ctx', '1782436e0ebf8d95ef4c086d94351698c464c4ee', 'DIRECT', 'none'], + ['agentmail-to/agentmail-plugins', '134887caf9375229415e09c760ae31baa4cc1ec3', 'DIRECT', 'none'], + ['ujjwalredd/sarathi', '08a51154a2f30af3eb4f6acb11115b9db912c5f8', 'DIRECT', 'none'], + ['sofus-nl/cc-plugin-codex', 'cc5123f7fa18db9c38f838a9b70119e5a0a6847c', 'DIRECT', 'none'], + ['RMI/speedy-skills', 'e983f800056a12b63fd60d5148538f98aaafe643', 'DIRECT', 'none'], + ['roadrunner-tuff/roadrunner-admin-plugin', '8e130c07656c8f9db8bf5431332c9aed60a4b133', 'DIRECT', 'none'] +]; + +const OH_MY_CASSETTE_MARKETPLACE = `{ + "name": "cassette-editor", + "interface": { + "displayName": "Cassette Editor" + }, + "plugins": [ + { + "name": "oh-my-cassette", + "source": { + "source": "url", + "url": "https://github.com/Cassette-Editor/oh-my-cassette.git", + "ref": "release" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_USE" + }, + "category": "Productivity" + } + ] +} +`; + +const FIXED_DETAILS = [ + ['bitrouter', '.', 'bitrouter', 'bitrouter', ['LICENSE'], ['skill', 'mcp']], + ['oh-my-cassette', '.', 'oh-my-cassette', 'cassette-editor', ['LICENSE'], ['skill', 'mcp']], + ['watercooler', '.', 'watercooler', 'watercooler', ['LICENSE'], ['skill', 'mcp']], + ['commercetools', '.', 'commercetools', 'commercetools', ['LICENSE'], ['skill', 'mcp']], + ['ctx', 'plugins/codex/ctx', 'ctx', 'ctx-local', ['LICENSE-APACHE', 'LICENSE-MIT'], ['skill', 'hook']], + ['agentmail', '.', 'agentmail', 'agentmail', ['LICENSE'], ['skill', 'mcp']], + ['sarathi', '.', 'sarathi', 'sarathi', ['LICENSE'], ['skill']], + [ + 'cc-plugin-codex', + '.', + 'cc-plugin-codex', + 'cc-plugin-codex', + ['LICENSE', 'plugins/cc-plugin-codex/LICENSE', 'plugins/cc-plugin-codex/NOTICE'], + ['skill', 'hook'] + ], + ['speedy-skills', '.', 'example-minimal', 'speedy-skills', ['LICENSE'], ['skill']], + ['roadrunner-admin', '.', 'roadrunner-admin', 'roadrunner', ['LICENSE'], ['skill', 'mcp']] +]; + +async function temporaryDirectory(t, prefix) { + const directory = await realpath(await mkdtemp(path.join(os.tmpdir(), prefix))); + t.after(() => rm(directory, { recursive: true, force: true })); + return directory; +} + +function publicReceipt(fixture, version, overrides = {}) { + return { + schemaVersion: '0.1.0', + status: 'PASS', + codexVersion: version, + platform: 'linux-x64', + plugin: { + name: fixture.plugin, + marketplace: fixture.marketplace, + sourceRoot: '/workspace' + }, + capabilities: fixture.expectedKinds.map((kind, index) => ({ + kind, + key: `${fixture.plugin}:${kind}:${index}`, + source: kind === 'skill' + ? 'plugin/read + skills/list' + : kind === 'hook' + ? 'plugin/read + hooks/list' + : 'plugin/read', + status: kind === 'skill' + ? 'DISCOVERED_EFFECTIVE' + : kind === 'hook' + ? 'DISCOVERED_UNTRUSTED' + : 'DECLARED_ONLY' + })), + isolation: { mode: 'strict', network: 'denied', hostState: 'denied' }, + ...overrides + }; +} + +function writeTarString(header, offset, length, value) { + Buffer.from(value).copy(header, offset, 0, length); +} + +function tarArchive(entries) { + const chunks = []; + for (const entry of entries) { + const data = Buffer.from(entry.data ?? ''); + const header = Buffer.alloc(512); + writeTarString(header, 0, 100, entry.path); + writeTarString(header, 100, 8, `${(entry.mode ?? 0o644).toString(8).padStart(7, '0')}\0`); + writeTarString(header, 108, 8, '0000000\0'); + writeTarString(header, 116, 8, '0000000\0'); + writeTarString(header, 124, 12, `${data.length.toString(8).padStart(11, '0')}\0`); + writeTarString(header, 136, 12, '00000000000\0'); + header.fill(0x20, 148, 156); + header[156] = (entry.type ?? '0').charCodeAt(0); + if (entry.linkPath) writeTarString(header, 157, 100, entry.linkPath); + writeTarString(header, 257, 6, 'ustar\0'); + writeTarString(header, 263, 2, '00'); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + writeTarString(header, 148, 8, `${checksum.toString(8).padStart(6, '0')}\0 `); + chunks.push(header, data); + const padding = (512 - (data.length % 512)) % 512; + if (padding > 0) chunks.push(Buffer.alloc(padding)); + } + chunks.push(Buffer.alloc(1024)); + return gzipSync(Buffer.concat(chunks)); +} + +function paxRecord(key, value) { + let length = Buffer.byteLength(` ${key}=${value}\n`) + 1; + while (true) { + const record = `${length} ${key}=${value}\n`; + const observed = Buffer.byteLength(record); + if (observed === length) return record; + length = observed; + } +} + +test('registry binds exactly the ten audited commits and only one static adapter', () => { + assert.deepEqual(CODEX_VERSIONS, ['0.147.0', '0.146.1']); + assert.deepEqual( + PUBLIC_FIXTURES.map(({ repository, commit, classification, adapterId }) => [ + repository, + commit, + classification, + adapterId + ]), + FIXED_IDENTITIES + ); + assert.equal(validateFixtureDefinitions(PUBLIC_FIXTURES), PUBLIC_FIXTURES); + assert.deepEqual( + PUBLIC_FIXTURES.map(({ + evidenceId, + marketplaceRoot, + plugin, + marketplace, + licensePaths, + expectedKinds + }) => [evidenceId, marketplaceRoot, plugin, marketplace, licensePaths, expectedKinds]), + FIXED_DETAILS + ); + for (const fixture of PUBLIC_FIXTURES) { + assert.equal(fixture.repositoryUrl, `https://github.com/${fixture.repository}`); + assert.equal( + fixture.archiveUrl, + `https://codeload.github.com/${fixture.repository}/tar.gz/${fixture.commit}` + ); + assert.match(fixture.archiveRoot, new RegExp(`-${fixture.commit}$`)); + } + + const changedCommit = structuredClone(PUBLIC_FIXTURES); + changedCommit[0].commit = '1111111111111111111111111111111111111111'; + assert.throws(() => validateFixtureDefinitions(changedCommit), /fixed fixture definition/i); + + const duplicate = structuredClone(PUBLIC_FIXTURES); + duplicate[9] = structuredClone(duplicate[0]); + assert.throws(() => validateFixtureDefinitions(duplicate), /fixed fixture definition|duplicate/i); + + const extraAdapter = structuredClone(PUBLIC_FIXTURES); + extraAdapter[0].adapterId = 'local-source-v1'; + assert.throws(() => validateFixtureDefinitions(extraAdapter), /fixed fixture definition|adapter/i); +}); + +test('environment accepts only exact released versions and a checkout-relative output root', () => { + const env = { + CODEX_CURRENT_VERSION: '0.147.0', + CODEX_PRIOR_VERSION: '0.146.1', + CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: 'artifacts/public-fixtures' + }; + assert.deepEqual(publicFixtureOptionsFromEnvironment(env, '/checkout'), { + architecture: process.arch, + outputBoundary: '/checkout', + outputRoot: '/checkout/artifacts/public-fixtures', + platform: process.platform + }); + + for (const invalid of [ + { ...env, CODEX_CURRENT_VERSION: 'latest' }, + { ...env, CODEX_PRIOR_VERSION: '0.146.0' }, + { ...env, CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: '/tmp/evidence' }, + { ...env, CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: '../evidence' }, + { ...env, CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: 'artifacts/../../evidence' } + ]) { + assert.throws( + () => publicFixtureOptionsFromEnvironment(invalid, '/checkout'), + /0\.147\.0|0\.146\.1|relative evidence directory/i + ); + } +}); + +test('archive entries must remain under one exact root and links may not escape it', () => { + const root = 'bitrouter-678384888b73fc290ce4ce503a8a7f2a5cbf6da8'; + const safe = [ + { path: `${root}/`, type: 'directory' }, + { path: `${root}/LICENSE`, type: 'file' }, + { path: `${root}/skills/current`, type: 'symlink', linkPath: '../LICENSE' } + ]; + assert.equal(validateArchiveEntries(safe, root), safe); + + for (const entries of [ + [{ path: `${root}/../escape`, type: 'file' }], + [{ path: `/absolute`, type: 'file' }], + [{ path: `${root}/safe\\..\\escape`, type: 'file' }], + [{ path: `other-root/LICENSE`, type: 'file' }], + [{ path: `${root}/skills/current`, type: 'symlink', linkPath: '../../../escape' }], + [{ path: `${root}/hard`, type: 'hardlink', linkPath: '../escape' }] + ]) { + assert.throws(() => validateArchiveEntries(entries, root), /archive|escape|root|path/i); + } +}); + +test('local-source-v1 changes only plugins/0/source after the immutable precondition', () => { + const adapted = applyLocalSourceAdapter(Buffer.from(OH_MY_CASSETTE_MARKETPLACE)); + assert.equal(adapted.originalSha256, 'd5c629f3a3b8dd2cdf560963c26b1c5e9dc062045fef13cafca178e7b1bd3d3f'); + assert.equal(adapted.adaptedSha256, 'bbecf8a43d3e993b8506f497a416a0cd83e535a18f3f647e79d6d2458fd6c7b1'); + const parsed = JSON.parse(adapted.bytes); + assert.deepEqual(parsed.plugins[0].source, { source: 'local', path: './' }); + const restored = structuredClone(parsed); + restored.plugins[0].source = { + source: 'url', + url: 'https://github.com/Cassette-Editor/oh-my-cassette.git', + ref: 'release' + }; + assert.deepEqual(restored, JSON.parse(OH_MY_CASSETTE_MARKETPLACE)); + assert.equal(createHash('sha256').update(adapted.bytes).digest('hex'), adapted.adaptedSha256); + + for (const invalid of [ + OH_MY_CASSETTE_MARKETPLACE.replace('"ref": "release"', '"ref": "main"'), + OH_MY_CASSETTE_MARKETPLACE.replace('"name": "oh-my-cassette"', '"name": "other"'), + JSON.stringify({ plugins: [] }) + ]) { + assert.throws(() => applyLocalSourceAdapter(Buffer.from(invalid)), /precondition|immutable/i); + } +}); + +test('receipt validator binds strict identity and audited capability semantics', () => { + const fixture = PUBLIC_FIXTURES[0]; + const base = publicReceipt(fixture, '0.147.0'); + assert.deepEqual( + validatePublicReceipt(base, { fixture, version: '0.147.0', architecture: 'x64' }), + base + ); + + const cases = [ + { ...base, status: 'FAIL' }, + { ...base, codexVersion: '0.146.1' }, + { ...base, platform: 'linux-arm64' }, + { ...base, plugin: { ...base.plugin, name: 'other' } }, + { ...base, plugin: { ...base.plugin, marketplace: 'other' } }, + { ...base, plugin: { ...base.plugin, sourceRoot: '/tmp/fixture' } }, + { ...base, isolation: { mode: 'env', network: 'not_enforced', hostState: 'not_enforced' } }, + { ...base, capabilities: base.capabilities.filter(({ kind }) => kind !== 'skill') }, + { + ...base, + capabilities: base.capabilities.map((capability) => capability.kind === 'mcp' + ? { ...capability, status: 'DISCOVERED_EFFECTIVE' } + : capability) + }, + { + ...base, + capabilities: [ + ...base.capabilities, + { kind: 'app', key: 'unexpected', source: 'plugin/read', status: 'DECLARED_ONLY' } + ] + } + ]; + for (const receipt of cases) { + assert.throws( + () => validatePublicReceipt(receipt, { + fixture, + version: '0.147.0', + architecture: 'x64' + }), + /receipt|pass|identity|capability|strict|platform/i + ); + } + assert.throws( + () => validatePublicReceipt({ + ...base, + capabilities: base.capabilities.map((capability, index) => index === 0 + ? { ...capability, key: '/Users/alice/private' } + : capability) + }, { + fixture, + version: '0.147.0', + architecture: 'x64', + personalMarkers: ['/Users/alice'] + }), + /personal|privacy|host path/i + ); + + const failed = { + ...base, + status: 'FAIL', + capabilities: base.capabilities.map((capability) => capability.kind === 'skill' + ? { ...capability, status: 'MISSING' } + : capability) + }; + assert.deepEqual(validatePublicReceipt(failed, { + fixture, + version: '0.147.0', + architecture: 'x64', + expectedExitCode: 1 + }), failed); +}); + +test('evidence paths are filenames under the output root, never absolute or traversing', () => { + for (const safe of ['bitrouter--0.147.0.json', 'public-fixture-summary.json']) { + assert.equal(validateRelativeEvidencePath(safe), safe); + } + for (const unsafe of [ + '/tmp/receipt.json', + '../receipt.json', + 'nested/receipt.json', + 'nested\\receipt.json', + '.', + '' + ]) { + assert.throws(() => validateRelativeEvidencePath(unsafe), /relative evidence path/i); + } +}); + +test('injected orchestration probes exactly ten by two cells and emits only sanitized evidence', async (t) => { + const boundary = await temporaryDirectory(t, 'public-fixture-boundary-'); + const outputRoot = path.join(boundary, 'artifacts', 'public-fixtures'); + const scratchParent = await temporaryDirectory(t, 'public-fixture-scratch-parent-'); + const scratchRoot = path.join(scratchParent, 'owned-run'); + const calls = []; + let cleanupCalls = 0; + + const summary = await runPublicFixtureMatrix({ + architecture: 'x64', + outputBoundary: boundary, + outputRoot, + personalMarkers: ['/Users/alice'], + platform: 'linux' + }, { + assertRuntime: async () => {}, + makeTemporaryRoot: async () => { + await mkdir(scratchRoot); + return scratchRoot; + }, + prepareFixture: async ({ fixture, temporaryRoot }) => { + const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); + await mkdir(marketplaceRoot); + return { + archiveSha256: 'a'.repeat(64), + checkoutSha256: 'b'.repeat(64), + marketplaceSha256: 'c'.repeat(64), + marketplaceRoot, + adapter: fixture.adapterId === 'none' + ? null + : { + id: 'local-source-v1', + originalSha256: 'd'.repeat(64), + adaptedSha256: 'e'.repeat(64) + } + }; + }, + probeCell: async ({ fixture, version }) => { + calls.push(`${fixture.repository}@${version}`); + return { code: 0, receipt: publicReceipt(fixture, version) }; + }, + removeTemporaryRoot: async (root) => { + cleanupCalls += 1; + await rm(root, { recursive: true, force: true }); + } + }); + + assert.equal(calls.length, 20); + assert.deepEqual(calls.slice(0, 4), [ + 'bitrouter/bitrouter@0.147.0', + 'bitrouter/bitrouter@0.146.1', + 'Cassette-Editor/oh-my-cassette@0.147.0', + 'Cassette-Editor/oh-my-cassette@0.146.1' + ]); + assert.equal(cleanupCalls, 1); + assert.equal(summary.status, 'PASS'); + assert.deepEqual(summary.gate, { + observed: 10, + required: 10, + cells: 20, + compatibleCells: 20, + incompatibleCells: 0 + }); + assert.equal(summary.outputRoot, '.'); + assert.equal(summary.fixtures.length, 10); + assert.equal(summary.fixtures.flatMap(({ receipts }) => receipts).length, 20); + + const names = (await readdir(outputRoot)).sort(); + assert.equal(names.length, 21); + assert.ok(names.includes('public-fixture-summary.json')); + for (const name of names) validateRelativeEvidencePath(name); + const serialized = await readFile(path.join(outputRoot, 'public-fixture-summary.json'), 'utf8'); + assert.equal(serialized.includes(boundary), false); + assert.equal(serialized.includes(scratchParent), false); + assert.equal(serialized.includes('/Users/alice'), false); + assert.deepEqual(JSON.parse(serialized), summary); +}); + +test('unexpected tool exits and cleanup failures fail closed without partial evidence', async (t) => { + for (const failure of ['tool', 'cleanup']) { + const boundary = await temporaryDirectory(t, `public-fixture-${failure}-boundary-`); + const outputRoot = path.join(boundary, 'artifacts', 'public-fixtures'); + const scratchParent = await temporaryDirectory(t, `public-fixture-${failure}-scratch-`); + const scratchRoot = path.join(scratchParent, 'owned-run'); + let probeCalls = 0; + await assert.rejects(runPublicFixtureMatrix({ + architecture: 'x64', + outputBoundary: boundary, + outputRoot, + platform: 'linux' + }, { + assertRuntime: async () => {}, + makeTemporaryRoot: async () => { + await mkdir(scratchRoot); + return scratchRoot; + }, + prepareFixture: async ({ fixture, temporaryRoot }) => { + const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); + await mkdir(marketplaceRoot); + return { + archiveSha256: 'a'.repeat(64), + checkoutSha256: 'b'.repeat(64), + marketplaceSha256: 'c'.repeat(64), + marketplaceRoot, + adapter: fixture.adapterId === 'none' + ? null + : { + id: 'local-source-v1', + originalSha256: 'd'.repeat(64), + adaptedSha256: 'e'.repeat(64) + } + }; + }, + probeCell: async ({ fixture, version }) => { + probeCalls += 1; + return failure === 'tool' + ? { code: 2, receipt: null, stderr: 'unexpected loader error' } + : { code: 0, receipt: publicReceipt(fixture, version) }; + }, + removeTemporaryRoot: async (root) => { + await rm(root, { recursive: true, force: true }); + if (failure === 'cleanup') throw new Error('fixture cleanup failed'); + } + }), failure === 'tool' ? /unexpected.*code 2|tool.*code 2/i : /cleanup failed/i); + assert.ok(probeCalls >= 1); + await assert.rejects(access(outputRoot), { code: 'ENOENT' }); + } +}); + +test('a valid strict FAIL receipt is retained as version/API incompatibility and holds the summary', async (t) => { + const boundary = await temporaryDirectory(t, 'public-fixture-incompatible-boundary-'); + const outputRoot = path.join(boundary, 'artifacts', 'public-fixtures'); + const scratchParent = await temporaryDirectory(t, 'public-fixture-incompatible-scratch-'); + const scratchRoot = path.join(scratchParent, 'owned-run'); + let first = true; + + const summary = await runPublicFixtureMatrix({ + architecture: 'x64', + outputBoundary: boundary, + outputRoot, + platform: 'linux' + }, { + assertRuntime: async () => {}, + makeTemporaryRoot: async () => { + await mkdir(scratchRoot); + return scratchRoot; + }, + prepareFixture: async ({ fixture, temporaryRoot }) => { + const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); + await mkdir(marketplaceRoot); + return { + archiveSha256: 'a'.repeat(64), + checkoutSha256: 'b'.repeat(64), + marketplaceSha256: 'c'.repeat(64), + marketplaceRoot, + adapter: fixture.adapterId === 'none' + ? null + : { + id: 'local-source-v1', + originalSha256: 'd'.repeat(64), + adaptedSha256: 'e'.repeat(64) + } + }; + }, + probeCell: async ({ fixture, version }) => { + const receipt = publicReceipt(fixture, version); + if (!first) return { code: 0, receipt }; + first = false; + return { + code: 1, + receipt: { + ...receipt, + status: 'FAIL', + capabilities: receipt.capabilities.map((capability) => capability.kind === 'skill' + ? { ...capability, status: 'MISSING' } + : capability) + } + }; + }, + removeTemporaryRoot: (root) => rm(root, { recursive: true, force: true }) + }); + + assert.equal(summary.status, 'HOLD'); + assert.equal(summary.gate.incompatibleCells, 1); + assert.equal(summary.gate.compatibleCells, 19); + assert.equal(summary.fixtures[0].receipts[0].outcome, 'VERSION_OR_API_INCOMPATIBLE'); + assert.equal(summary.fixtures[0].receipts[1].outcome, 'PASS'); + assert.equal( + JSON.parse(await readFile( + path.join(outputRoot, summary.fixtures[0].receipts[0].receiptPath), + 'utf8' + )).status, + 'FAIL' + ); +}); + +test('evidence root symlinks are rejected before writes and outside state remains unchanged', async (t) => { + const boundary = await temporaryDirectory(t, 'public-fixture-symlink-boundary-'); + const outside = await temporaryDirectory(t, 'public-fixture-symlink-outside-'); + const sentinel = path.join(outside, 'sentinel.txt'); + await writeFile(sentinel, 'unchanged\n'); + await symlink(outside, path.join(boundary, 'artifacts'), 'dir'); + let prepared = false; + + await assert.rejects(runPublicFixtureMatrix({ + architecture: 'x64', + outputBoundary: boundary, + outputRoot: path.join(boundary, 'artifacts', 'public-fixtures'), + platform: 'linux' + }, { + assertRuntime: async () => {}, + makeTemporaryRoot: async () => { prepared = true; } + }), /symlink/i); + assert.equal(prepared, false); + assert.equal(await readFile(sentinel, 'utf8'), 'unchanged\n'); + await assert.rejects(access(path.join(outside, 'public-fixtures')), { code: 'ENOENT' }); +}); + +test('swapping the created evidence root for a symlink cannot write outside', async (t) => { + const boundary = await temporaryDirectory(t, 'public-fixture-swap-boundary-'); + const outside = await temporaryDirectory(t, 'public-fixture-swap-outside-'); + const outputRoot = path.join(boundary, 'artifacts', 'public-fixtures'); + const scratchParent = await temporaryDirectory(t, 'public-fixture-swap-scratch-'); + const scratchRoot = path.join(scratchParent, 'owned-run'); + const sentinel = path.join(outside, 'sentinel.txt'); + await writeFile(sentinel, 'unchanged\n'); + let swapped = false; + + await assert.rejects(runPublicFixtureMatrix({ + architecture: 'x64', + outputBoundary: boundary, + outputRoot, + platform: 'linux' + }, { + assertRuntime: async () => {}, + makeTemporaryRoot: async () => { + await mkdir(scratchRoot); + return scratchRoot; + }, + prepareFixture: async ({ fixture, temporaryRoot }) => { + const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); + await mkdir(marketplaceRoot); + return { + archiveSha256: 'a'.repeat(64), + checkoutSha256: 'b'.repeat(64), + marketplaceSha256: 'c'.repeat(64), + marketplaceRoot, + adapter: null + }; + }, + probeCell: async ({ fixture, version }) => { + if (!swapped) { + swapped = true; + await rm(outputRoot, { recursive: true }); + await symlink(outside, outputRoot, 'dir'); + } + return { code: 0, receipt: publicReceipt(fixture, version) }; + }, + removeTemporaryRoot: (root) => rm(root, { recursive: true, force: true }) + }), /evidence|symlink|cleanup/i); + + assert.equal(await readFile(sentinel, 'utf8'), 'unchanged\n'); + assert.deepEqual(await readdir(outside), ['sentinel.txt']); +}); + +test('archive fetch uses only exact credential-free codeload and enforces byte bounds', async () => { + const fixture = PUBLIC_FIXTURES[0]; + let observed; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(Uint8Array.from([1, 2])); + controller.enqueue(Uint8Array.from([3, 4])); + controller.close(); + } + }); + const fetched = await fetchPublicArchive(fixture, { + fetchImpl: async (url, options) => { + observed = { url, options }; + return { + ok: true, + status: 200, + url: fixture.archiveUrl, + headers: { get: (name) => name.toLowerCase() === 'content-length' ? '4' : null }, + body + }; + }, + maxBytes: 4, + timeoutMs: 100 + }); + assert.deepEqual(fetched.bytes, Buffer.from([1, 2, 3, 4])); + assert.equal(fetched.archiveSha256, '9f64a747e1b97f131fabb6b447296c9b6f0201e79fb3c5356e6c77e89b6a806a'); + assert.equal(observed.url, fixture.archiveUrl); + assert.equal(observed.options.redirect, 'error'); + assert.equal(observed.options.credentials, 'omit'); + assert.ok(observed.options.signal instanceof AbortSignal); + assert.deepEqual(Object.keys(observed.options.headers).sort(), ['accept', 'user-agent']); + + await assert.rejects(fetchPublicArchive(fixture, { + fetchImpl: async () => ({ + ok: true, + status: 200, + url: fixture.archiveUrl, + headers: { get: () => '5' }, + body: new ReadableStream({ start(controller) { controller.close(); } }) + }), + maxBytes: 4, + timeoutMs: 100 + }), /archive.*size|exceed/i); + + await assert.rejects(fetchPublicArchive(fixture, { + fetchImpl: async () => ({ + ok: true, + status: 200, + url: 'https://evil.example/archive.tar.gz', + headers: { get: () => '1' }, + body: new ReadableStream({ + start(controller) { + controller.enqueue(Uint8Array.from([1])); + controller.close(); + } + }) + }), + maxBytes: 4, + timeoutMs: 100 + }), /codeload|identity|url/i); +}); + +test('tar parser validates before manual extraction and never preserves executable bits', async (t) => { + const fixture = PUBLIC_FIXTURES[6]; + const root = fixture.archiveRoot; + const archive = tarArchive([ + { path: `${root}/`, type: '5', mode: 0o755 }, + { path: `${root}/LICENSE`, data: 'MIT\n' }, + { path: `${root}/run-me.sh`, data: '#!/bin/sh\ntouch EXECUTED\n', mode: 0o755 }, + { path: `${root}/license-link`, type: '2', linkPath: 'LICENSE' } + ]); + const entries = parseTarGzipArchive(archive, { + expectedRoot: root, + maxUncompressedBytes: 1024 * 1024 + }); + assert.deepEqual(entries.map(({ path: entryPath, type }) => [entryPath, type]), [ + [`${root}/`, 'directory'], + [`${root}/LICENSE`, 'file'], + [`${root}/run-me.sh`, 'file'], + [`${root}/license-link`, 'symlink'] + ]); + + const destination = await temporaryDirectory(t, 'public-fixture-extract-'); + const checkoutRoot = await extractArchiveEntries(entries, destination, root); + assert.equal(await readFile(path.join(checkoutRoot, 'LICENSE'), 'utf8'), 'MIT\n'); + const scriptMode = await lstat(path.join(checkoutRoot, 'run-me.sh')); + assert.equal(scriptMode.mode & 0o111, 0); + await auditExtractedSymlinks(checkoutRoot); + await assert.rejects(access(path.join(checkoutRoot, 'EXECUTED')), { code: 'ENOENT' }); + + for (const unsafe of [ + tarArchive([{ path: `${root}/../escape`, data: 'bad' }]), + tarArchive([{ path: `${root}/link`, type: '2', linkPath: '../../escape' }]) + ]) { + assert.throws( + () => parseTarGzipArchive(unsafe, { + expectedRoot: root, + maxUncompressedBytes: 1024 * 1024 + }), + /archive|escape|root|path/i + ); + } + + const corrupt = Buffer.from(archive); + corrupt[20] ^= 0xff; + assert.throws( + () => parseTarGzipArchive(corrupt, { + expectedRoot: root, + maxUncompressedBytes: 1024 * 1024 + }), + /archive|gzip|checksum|invalid/i + ); +}); + +test('tar parser resolves bounded PAX path metadata without exposing metadata entries', () => { + const fixture = PUBLIC_FIXTURES[6]; + const longPath = `${fixture.archiveRoot}/${'nested-'.repeat(16)}skill.md`; + const archive = tarArchive([ + { + path: 'pax_global_header', + type: 'g', + data: paxRecord('comment', fixture.commit) + }, + { + path: `${fixture.archiveRoot}/pax-header`, + type: 'x', + data: paxRecord('path', longPath) + }, + { path: `${fixture.archiveRoot}/placeholder`, data: 'content\n' } + ]); + const entries = parseTarGzipArchive(archive, { + expectedRoot: fixture.archiveRoot, + maxUncompressedBytes: 1024 * 1024 + }); + assert.equal(entries.length, 1); + assert.equal(entries[0].path, longPath); + assert.equal(entries[0].data.toString('utf8'), 'content\n'); +}); + +test('on-disk symlink audit rejects a lexical escape from the checkout', async (t) => { + const checkout = await temporaryDirectory(t, 'public-fixture-checkout-'); + const outside = await temporaryDirectory(t, 'public-fixture-checkout-outside-'); + await symlink(outside, path.join(checkout, 'escape'), 'dir'); + await assert.rejects(auditExtractedSymlinks(checkout), /symlink.*escape|escape.*symlink/i); +}); + +test('real preparation verifies licenses, hashes source, and applies only the audited adapter', async (t) => { + const direct = PUBLIC_FIXTURES[6]; + const directArchive = tarArchive([ + { path: `${direct.archiveRoot}/`, type: '5' }, + { path: `${direct.archiveRoot}/LICENSE`, data: 'MIT\n' }, + { path: `${direct.archiveRoot}/.agents/`, type: '5' }, + { path: `${direct.archiveRoot}/.agents/plugins/`, type: '5' }, + { path: `${direct.archiveRoot}/.agents/plugins/marketplace.json`, data: '{}\n' } + ]); + const directTemporaryRoot = await temporaryDirectory(t, 'public-fixture-prepare-direct-'); + const directPrepared = await preparePublicFixture({ + fixture: direct, + temporaryRoot: directTemporaryRoot + }, { + fetchArchive: async () => ({ + bytes: directArchive, + archiveSha256: createHash('sha256').update(directArchive).digest('hex') + }) + }); + assert.equal(directPrepared.archiveSha256, createHash('sha256').update(directArchive).digest('hex')); + assert.match(directPrepared.checkoutSha256, /^[a-f0-9]{64}$/); + assert.match(directPrepared.marketplaceSha256, /^[a-f0-9]{64}$/); + assert.equal(directPrepared.adapter, null); + assert.equal(await readFile(path.join(directPrepared.marketplaceRoot, 'LICENSE'), 'utf8'), 'MIT\n'); + + const adapted = PUBLIC_FIXTURES[1]; + const adaptedArchive = tarArchive([ + { path: `${adapted.archiveRoot}/`, type: '5' }, + { path: `${adapted.archiveRoot}/LICENSE`, data: 'MIT\n' }, + { path: `${adapted.archiveRoot}/.agents/`, type: '5' }, + { path: `${adapted.archiveRoot}/.agents/plugins/`, type: '5' }, + { + path: `${adapted.archiveRoot}/.agents/plugins/marketplace.json`, + data: OH_MY_CASSETTE_MARKETPLACE + } + ]); + const adaptedTemporaryRoot = await temporaryDirectory(t, 'public-fixture-prepare-adapted-'); + const adaptedPrepared = await preparePublicFixture({ + fixture: adapted, + temporaryRoot: adaptedTemporaryRoot + }, { + fetchArchive: async () => ({ + bytes: adaptedArchive, + archiveSha256: createHash('sha256').update(adaptedArchive).digest('hex') + }) + }); + assert.deepEqual(adaptedPrepared.adapter, { + id: 'local-source-v1', + originalSha256: 'd5c629f3a3b8dd2cdf560963c26b1c5e9dc062045fef13cafca178e7b1bd3d3f', + adaptedSha256: 'bbecf8a43d3e993b8506f497a416a0cd83e535a18f3f647e79d6d2458fd6c7b1' + }); + assert.deepEqual( + JSON.parse(await readFile( + path.join(adaptedPrepared.marketplaceRoot, '.agents/plugins/marketplace.json'), + 'utf8' + )).plugins[0].source, + { source: 'local', path: './' } + ); + + const missingLicense = tarArchive([ + { path: `${direct.archiveRoot}/`, type: '5' }, + { path: `${direct.archiveRoot}/README.md`, data: 'no license\n' } + ]); + const missingTemporaryRoot = await temporaryDirectory(t, 'public-fixture-prepare-missing-'); + await assert.rejects(preparePublicFixture({ + fixture: direct, + temporaryRoot: missingTemporaryRoot + }, { + fetchArchive: async () => ({ + bytes: missingLicense, + archiveSha256: createHash('sha256').update(missingLicense).digest('hex') + }) + }), /license/i); + + const adapterSymlink = tarArchive([ + { path: `${adapted.archiveRoot}/`, type: '5' }, + { path: `${adapted.archiveRoot}/LICENSE`, data: OH_MY_CASSETTE_MARKETPLACE }, + { path: `${adapted.archiveRoot}/.agents/`, type: '5' }, + { path: `${adapted.archiveRoot}/.agents/plugins/`, type: '5' }, + { + path: `${adapted.archiveRoot}/.agents/plugins/marketplace.json`, + type: '2', + linkPath: '../../LICENSE' + } + ]); + const symlinkTemporaryRoot = await temporaryDirectory(t, 'public-fixture-prepare-symlink-'); + await assert.rejects(preparePublicFixture({ + fixture: adapted, + temporaryRoot: symlinkTemporaryRoot + }, { + fetchArchive: async () => ({ + bytes: adapterSymlink, + archiveSha256: createHash('sha256').update(adapterSymlink).digest('hex') + }) + }), /adapter.*regular|regular.*adapter|symlink/i); +}); + +test('production cell probe invokes the real CLI contract with a fresh staged receipt then removes it', async (t) => { + const temporaryRoot = await temporaryDirectory(t, 'public-fixture-probe-'); + const marketplaceRoot = path.join(temporaryRoot, 'marketplace'); + await mkdir(marketplaceRoot); + const fixture = PUBLIC_FIXTURES[0]; + const calls = []; + + const result = await probePublicFixtureCell({ + architecture: 'x64', + fixture, + marketplaceRoot, + temporaryRoot, + version: '0.147.0' + }, { + cliMain: async (argv) => { + calls.push(argv); + const output = argv[argv.indexOf('--output') + 1]; + await writeFile(output, `${JSON.stringify(publicReceipt(fixture, '0.147.0'))}\n`, { + flag: 'wx' + }); + return 0; + } + }); + + assert.equal(result.code, 0); + assert.deepEqual(result.receipt, publicReceipt(fixture, '0.147.0')); + assert.deepEqual(calls, [[ + '--marketplace-root', marketplaceRoot, + '--plugin', 'bitrouter', + '--codex-version', '0.147.0', + '--cwd', marketplaceRoot, + '--output', path.join(temporaryRoot, 'staged-receipts', 'bitrouter--codex-0.147.0.json'), + '--isolation', 'strict', + '--quiet' + ]]); + assert.deepEqual(await readdir(path.join(temporaryRoot, 'staged-receipts')), []); +}); + +test('production cell probe binds code one to a valid FAIL and rejects code two or stale output', async (t) => { + const temporaryRoot = await temporaryDirectory(t, 'public-fixture-probe-errors-'); + const marketplaceRoot = path.join(temporaryRoot, 'marketplace'); + await mkdir(marketplaceRoot); + const fixture = PUBLIC_FIXTURES[0]; + const failed = publicReceipt(fixture, '0.147.0'); + failed.status = 'FAIL'; + failed.capabilities = failed.capabilities.map((capability) => capability.kind === 'skill' + ? { ...capability, status: 'MISSING' } + : capability); + + const validFailure = await probePublicFixtureCell({ + architecture: 'x64', + fixture, + marketplaceRoot, + temporaryRoot, + version: '0.147.0' + }, { + cliMain: async (argv) => { + await writeFile(argv[argv.indexOf('--output') + 1], `${JSON.stringify(failed)}\n`, { + flag: 'wx' + }); + return 1; + } + }); + assert.equal(validFailure.code, 1); + assert.equal(validFailure.receipt.status, 'FAIL'); + + await assert.rejects(probePublicFixtureCell({ + architecture: 'x64', + fixture, + marketplaceRoot, + temporaryRoot, + version: '0.146.1' + }, { + cliMain: async (argv, io) => { + io.stderr.write('unexpected loader error\n'); + await writeFile( + argv[argv.indexOf('--output') + 1], + `${JSON.stringify(publicReceipt(fixture, '0.146.1'))}\n`, + { flag: 'wx' } + ); + return 2; + } + }), /code 2|loader error/i); + assert.deepEqual(await readdir(path.join(temporaryRoot, 'staged-receipts')), []); + + const stalePath = path.join( + temporaryRoot, + 'staged-receipts', + 'bitrouter--codex-0.146.1.json' + ); + await writeFile(stalePath, 'stale\n', { flag: 'wx' }); + let invoked = false; + await assert.rejects(probePublicFixtureCell({ + architecture: 'x64', + fixture, + marketplaceRoot, + temporaryRoot, + version: '0.146.1' + }, { + cliMain: async () => { invoked = true; } + }), /fresh|already exists|staged/i); + assert.equal(invoked, false); + assert.equal(await readFile(stalePath, 'utf8'), 'stale\n'); +}); + +test('runtime preflight hard-fails outside Linux and requires a live Docker server', async () => { + let inspections = 0; + await assert.rejects(assertPublicFixtureRuntime({ platform: 'darwin' }, { + inspectDocker: async () => { inspections += 1; } + }), /requires Linux.*darwin/i); + assert.equal(inspections, 0); + + await assertPublicFixtureRuntime({ platform: 'linux' }, { + inspectDocker: async () => { + inspections += 1; + return { code: 0, stdout: '27.5.1\n', stderr: '' }; + } + }); + assert.equal(inspections, 1); + + for (const result of [ + { code: 1, stdout: '', stderr: 'daemon unavailable' }, + { code: 0, stdout: '', stderr: '' } + ]) { + await assert.rejects(assertPublicFixtureRuntime({ platform: 'linux' }, { + inspectDocker: async () => result + }), /requires Docker|daemon unavailable/i); + } +}); + +test('atomic evidence creation never overwrites an existing receipt and leaves no temp file', async (t) => { + const boundary = await temporaryDirectory(t, 'public-fixture-atomic-boundary-'); + const outputRoot = path.join(boundary, 'evidence'); + await mkdir(outputRoot); + await writeAtomicEvidenceJson( + { boundary, outputRoot }, + 'receipt.json', + { status: 'PASS' } + ); + assert.deepEqual(JSON.parse(await readFile(path.join(outputRoot, 'receipt.json'), 'utf8')), { + status: 'PASS' + }); + await assert.rejects(writeAtomicEvidenceJson( + { boundary, outputRoot }, + 'receipt.json', + { status: 'FAIL' } + ), /exist|fresh|atomic/i); + assert.deepEqual(JSON.parse(await readFile(path.join(outputRoot, 'receipt.json'), 'utf8')), { + status: 'PASS' + }); + assert.deepEqual(await readdir(outputRoot), ['receipt.json']); +}); + +test('entrypoint binds exact environment options and reports failures with exit code two', async () => { + const env = { + CODEX_CURRENT_VERSION: '0.147.0', + CODEX_PRIOR_VERSION: '0.146.1', + CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: 'artifacts/public-fixtures' + }; + let observedOptions; + let stderr = ''; + const code = await publicFixtureMain({ + cwd: '/checkout', + env, + stderr: { write: (value) => { stderr += String(value); } } + }, { + runMatrix: async (options) => { + observedOptions = options; + return { status: 'PASS' }; + } + }); + assert.equal(code, 0); + assert.equal(stderr, ''); + assert.deepEqual(observedOptions, { + architecture: process.arch, + outputBoundary: '/checkout', + outputRoot: '/checkout/artifacts/public-fixtures', + platform: process.platform + }); + + const holdCode = await publicFixtureMain({ + cwd: '/checkout', + env, + stderr: { write: (value) => { stderr += String(value); } } + }, { + runMatrix: async () => ({ status: 'HOLD' }) + }); + assert.equal(holdCode, 1, 'a classified incompatibility must not make CI green'); + + const toolErrorCode = await publicFixtureMain({ + cwd: '/checkout', + env, + stderr: { write: (value) => { stderr += String(value); } } + }, { + runMatrix: async () => { throw new Error('Docker unavailable'); } + }); + assert.equal(toolErrorCode, 2); + assert.match(stderr, /Error: Docker unavailable/i); + + let invoked = false; + const invalidCode = await publicFixtureMain({ + cwd: '/checkout', + env: { ...env, CODEX_CURRENT_VERSION: 'latest' }, + stderr: { write: (value) => { stderr += String(value); } } + }, { + runMatrix: async () => { invoked = true; } + }); + assert.equal(invalidCode, 2); + assert.equal(invoked, false); + assert.match(stderr, /Error:.*0\.147\.0/i); +}); From f0b13add512f195f1d8a5ff8e00dbce59704b9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=E1=BB=93=20Kh=E1=BA=AFc=20Huy?= <256174233+builtbyhuy@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:50:29 +0700 Subject: [PATCH 7/8] docs: record public fixture diagnostics --- docs/evidence/public-fixture-matrix.md | 37 +++++++++++++++++++ .../plans/2026-08-10-codex-plugin-check.md | 21 ++++++----- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/docs/evidence/public-fixture-matrix.md b/docs/evidence/public-fixture-matrix.md index 5523fc8..35f640b 100644 --- a/docs/evidence/public-fixture-matrix.md +++ b/docs/evidence/public-fixture-matrix.md @@ -56,6 +56,43 @@ precedence, and the unmodified fixture passes on both versions. These environment observations helped falsify the implementation. They do not satisfy the strict 10-repository gate. +After the fixed runner was implemented, a complete local environment matrix +ran all 20 repository/version cells against the prepared `0.147.0` and +`0.146.1` binaries. All 20 returned exit `0` and `PASS` with exact +repository/plugin/marketplace/version identities. Every declared skill was +`DISCOVERED_EFFECTIVE`; the `ctx` and `cc-plugin-codex` hooks were +`DISCOVERED_UNTRUSTED`; every MCP declaration was `DECLARED_ONLY`. No app or +unexpected capability kind appeared. The temporary checkouts and receipts were +removed after reconciliation. + +That result is a full loader diagnostic, but its `env` isolation does not deny +network or personal host state. It is not counted in the strict result column. + +On 2026-08-10, the production preparation path fetched, parsed, extracted, +license-checked, hashed and cleaned all ten immutable archives without running +Docker or any upstream code. A second root-agent run reproduced all ten archive +hashes: + +| Fixture | Archive SHA-256 | Extracted checkout SHA-256 | Probed marketplace SHA-256 | +| --- | --- | --- | --- | +| `bitrouter` | `cd173128072bc769995a46baf1c9b19b0215214e8719a2f3779d4e3d52a69351` | `32e3cf5e9211dc28c55bd4bc2e7b794a73d9c961805aa1cc8436851ae17bff73` | `32e3cf5e9211dc28c55bd4bc2e7b794a73d9c961805aa1cc8436851ae17bff73` | +| `oh-my-cassette` | `3d64d09b7fae024d53616d4b04173e1a584f7982d3971fb94d0ea54cfed37287` | `86f6123019a98f6e59e0a0b3c0a41759aa52982f3af3242961b7be5cfbcae4d3` | `6218b332623998fb9cb9b116614d2465e485b376bdc930976bee556ab417e8e4` | +| `watercooler` | `aed68325b301e45b422f491865e7c3325c53d966dc89d3294deba45fa280ba3c` | `7f40c2886ade5da2325c94852db6062fa4a95c0b76ae868cf9aa035e28da5420` | `7f40c2886ade5da2325c94852db6062fa4a95c0b76ae868cf9aa035e28da5420` | +| `commercetools` | `7e4ac439b75a064a08a6e2a107d1ea8bb0221b974807670cea56c63a6cfd9094` | `8620f6238f94edeebca2eafa8fe193d0f7694841740d0c30b226fbfe6783eacf` | `8620f6238f94edeebca2eafa8fe193d0f7694841740d0c30b226fbfe6783eacf` | +| `ctx` | `5ef97584aacb6874cede5780ee47d137597a6ebd1fd3ecf04f0e97c79dfd8dfd` | `78bedfa1c634771af25f27db4205079b973fa05ee58a4f17442595a08d406f48` | `552f221e0c7c8ba59956829b14e5ee5b1a242c81d72d533e1f6d9b1a5f805c93` | +| `agentmail` | `ec595bfaf2e7201ade5e8c5902b424caa05948d1c6558e3f53be3468f02eca14` | `eac1dc131996059547801663bc7cd36c38221b0b28d9012c062c4af441e66800` | `eac1dc131996059547801663bc7cd36c38221b0b28d9012c062c4af441e66800` | +| `sarathi` | `f4577e9777d44111b1074460eba28987c80db2296f1983a018b391d99d42e7d8` | `7b45528f5816ed23191abf7aaae2670bd342989f6774007eb07937665bbcdc02` | `7b45528f5816ed23191abf7aaae2670bd342989f6774007eb07937665bbcdc02` | +| `cc-plugin-codex` | `57cff2045571f47a75231700f496f4aaaa4d780e1904721e5679a5d68f420cc9` | `562c1c772a33588dd7364ce11316bac0225c135bb84da91a4fdb619c32e3ba74` | `562c1c772a33588dd7364ce11316bac0225c135bb84da91a4fdb619c32e3ba74` | +| `speedy-skills` | `6734bb6adb707b623cbb22a9a8c12c571d7fbe51b768cc066055aa86da6e6da0` | `60459f76469df1ae420fab647a91b4413b79310d6166921a633877bd1e84fc9c` | `60459f76469df1ae420fab647a91b4413b79310d6166921a633877bd1e84fc9c` | +| `roadrunner-admin` | `a54ddc226d7f2af17d867b82dd83ff0dfa330abb9dc5e6f80b3cf0d639ad4650` | `1b057c8ac8031ea74a02a5ee46cd716f3f020df3779d37c32f3f47ede6bb0b5b` | `1b057c8ac8031ea74a02a5ee46cd716f3f020df3779d37c32f3f47ede6bb0b5b` | + +For `oh-my-cassette`, the observed adapter input is +`d5c629f3a3b8dd2cdf560963c26b1c5e9dc062045fef13cafca178e7b1bd3d3f` +and its adapted output is +`bbecf8a43d3e993b8506f497a416a0cd83e535a18f3f647e79d6d2458fd6c7b1`. +The differing marketplace hash is therefore expected and bounded. These are +preparation receipts, not Codex compatibility receipts. + ## Publication gate Change this decision from `HOLD` only after all 20 strict cells are retained, diff --git a/docs/superpowers/plans/2026-08-10-codex-plugin-check.md b/docs/superpowers/plans/2026-08-10-codex-plugin-check.md index f6663be..4d0d3f5 100644 --- a/docs/superpowers/plans/2026-08-10-codex-plugin-check.md +++ b/docs/superpowers/plans/2026-08-10-codex-plugin-check.md @@ -491,7 +491,7 @@ git commit -m "docs: prepare evidence-bound v0 release" No repository script, build, dependency install, hook, MCP server, app, or model is executed. No upstream source file is published in the artifact. -- [ ] **Step 1: Write failing safety, identity, adapter, and ledger tests** +- [x] **Step 1: Write failing safety, identity, adapter, and ledger tests** Bind exactly ten unique HTTPS GitHub repositories, full commit SHAs, expected plugin IDs, marketplace roots, licenses, and adapter IDs. Reject mutable refs, @@ -499,22 +499,23 @@ path escapes, duplicate rows, extra adapter fields, untrusted workflow events, receipt identity drift, missing expected capability kinds, absolute evidence paths, and arbitrary tool errors mislabeled as incompatibility. -- [ ] **Step 2: Verify RED** +- [x] **Step 2: Verify RED** Run: `node --test test/public-fixtures.test.mjs test/workflow-contract.test.mjs` Expected: FAIL because the fixed runner and trusted workflow do not exist. -- [ ] **Step 3: Implement the bounded fixed runner** +- [x] **Step 3: Implement the bounded fixed runner** -Fetch and verify each exact commit with an isolated Git configuration, remove -only owned Git metadata before probing, preserve upstream license/notice files, -apply the single audited static adapter, and run the production strict CLI for -all 20 repository/version cells. Validate every receipt with the production -schema and exact source/plugin/platform/isolation expectations. A conformance -receipt may truthfully be `FAIL`; an unexpected tool error fails the matrix. +Fetch each exact commit from its credential-free codeload archive, validate and +manually extract the bounded tar entries into owned state, preserve upstream +license/notice files, apply the single audited static adapter, and run the +production strict CLI for all 20 repository/version cells. Validate every +receipt with the production schema and exact source/plugin/platform/isolation +expectations. A conformance receipt may truthfully be `FAIL`; an unexpected +tool error fails the matrix. -- [ ] **Step 4: Run non-certifying local environment probes** +- [x] **Step 4: Run non-certifying local environment probes** Use the prepared released `0.147.0` and `0.146.1` binaries to reconcile each observed capability set against the immutable source before strict CI. Record From d6e47bbcef08717836315af6e5d1e907be002a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=E1=BB=93=20Kh=E1=BA=AFc=20Huy?= <256174233+builtbyhuy@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:13:56 +0700 Subject: [PATCH 8/8] fix: bind public fixture evidence --- .github/workflows/public-fixtures.yml | 2 +- README.md | 18 +- docs/evidence/public-fixture-matrix.md | 87 ++-- docs/evidence/technical-falsifier.md | 2 +- .../plans/2026-08-10-codex-plugin-check.md | 11 +- scripts/falsify-public-fixtures.mjs | 470 ++++++++++++++++-- src/check-plugin.mjs | 33 +- src/cli.mjs | 43 ++ test/check-plugin.test.mjs | 28 ++ test/cli.test.mjs | 25 +- test/public-fixtures.test.mjs | 447 ++++++++++++++--- test/workflow-contract.test.mjs | 1 + 12 files changed, 997 insertions(+), 170 deletions(-) diff --git a/.github/workflows/public-fixtures.yml b/.github/workflows/public-fixtures.yml index f34389d..b6517f4 100644 --- a/.github/workflows/public-fixtures.yml +++ b/.github/workflows/public-fixtures.yml @@ -51,6 +51,6 @@ jobs: name: public-fixture-evidence-${{ github.run_id }}-${{ github.run_attempt }} path: artifacts/public-fixtures if-no-files-found: error - retention-days: 14 + retention-days: 90 compression-level: 9 include-hidden-files: false diff --git a/README.md b/README.md index cae44de..cf84132 100644 --- a/README.md +++ b/README.md @@ -31,11 +31,13 @@ about their effective runtime behavior. ## No-execution boundary -The probe never sends a model request and never executes plugin hooks, MCP -servers, plugin scripts, apps, or authentication flows. It disables remote +The checker never sends a model request or asks Codex to execute plugin hooks, +MCP servers, plugin scripts, apps, or authentication flows. It disables remote plugin discovery and does not mount personal Codex or agent state into strict -mode. Synthetic hook and MCP commands are execution sentinels: if Codex starts -either command during the falsifier, the run fails before isolation cleanup. +mode. In the exact synthetic fixture, hook and MCP commands are execution +sentinels: if Codex starts either command during the falsifier, the run fails +before isolation cleanup. Third-party fixture checks rely on this bounded API +path and do not claim universal per-fixture execution tracing. Network access is permitted only while Docker prepares an image containing the exact released Codex package. The complete strict probe then runs once with @@ -60,9 +62,11 @@ release reference exists while the repository is on HOLD. Once the release gates pass, consumers should pin the Action to a reviewed full commit SHA—not a floating branch or tag. -Action inputs mirror the CLI: `marketplace-root`, `plugin`, `codex-version`, -`codex`, `cwd`, `output`, and `isolation`. The outputs are `status`, the full -`receipt`, and the observed `codex-version`. +Action inputs cover the general check: `marketplace-root`, `plugin`, +`codex-version`, `codex`, `cwd`, `output`, and `isolation`. The fixed public +matrix additionally uses CLI-only `expected-plugin-root` and +`expected-plugin-version` gates. Action outputs are `status`, the full receipt, +and the observed `codex-version`. ## CLI diagnostic example diff --git a/docs/evidence/public-fixture-matrix.md b/docs/evidence/public-fixture-matrix.md index 35f640b..cd2cc24 100644 --- a/docs/evidence/public-fixture-matrix.md +++ b/docs/evidence/public-fixture-matrix.md @@ -11,18 +11,18 @@ it does not turn a static compatibility expectation into a runtime result. Each repository is fetched at one full commit SHA without credentials. The target matrix is Codex `0.147.0` and `0.146.1`, for 20 independent strict cells. -| Repository | Commit | Marketplace root | Plugin | Preparation | Expected kinds | License | Strict result | -| --- | --- | --- | --- | --- | --- | --- | --- | -| [`bitrouter/bitrouter`](https://github.com/bitrouter/bitrouter) | [`678384888b73fc290ce4ce503a8a7f2a5cbf6da8`](https://github.com/bitrouter/bitrouter/commit/678384888b73fc290ce4ce503a8a7f2a5cbf6da8) | `.` | `bitrouter` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | -| [`Cassette-Editor/oh-my-cassette`](https://github.com/Cassette-Editor/oh-my-cassette) | [`cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32`](https://github.com/Cassette-Editor/oh-my-cassette/commit/cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32) | `.` | `oh-my-cassette` | `STATIC_ADAPTER:local-source-v1` | skill, MCP | MIT | `UNRUN` | -| [`mostlyharmless-ai/watercooler`](https://github.com/mostlyharmless-ai/watercooler) | [`a5efa89df02e7796e20881fef4847f129d84d367`](https://github.com/mostlyharmless-ai/watercooler/commit/a5efa89df02e7796e20881fef4847f129d84d367) | `.` | `watercooler` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | -| [`commercetools/commercetools-ai-plugins`](https://github.com/commercetools/commercetools-ai-plugins) | [`440d6bd56eb2969b6a0dd41e3fcff286def0787c`](https://github.com/commercetools/commercetools-ai-plugins/commit/440d6bd56eb2969b6a0dd41e3fcff286def0787c) | `.` | `commercetools` | `DIRECT` | skill, MCP | CC-BY-4.0 | `UNRUN` | -| [`agentis-tools/ctx`](https://github.com/agentis-tools/ctx) | [`1782436e0ebf8d95ef4c086d94351698c464c4ee`](https://github.com/agentis-tools/ctx/commit/1782436e0ebf8d95ef4c086d94351698c464c4ee) | `plugins/codex/ctx` | `ctx` | `DIRECT` | skill, hook | Apache-2.0 OR MIT | `UNRUN` | -| [`agentmail-to/agentmail-plugins`](https://github.com/agentmail-to/agentmail-plugins) | [`134887caf9375229415e09c760ae31baa4cc1ec3`](https://github.com/agentmail-to/agentmail-plugins/commit/134887caf9375229415e09c760ae31baa4cc1ec3) | `.` | `agentmail` | `DIRECT` | skill, MCP | MIT | `UNRUN` | -| [`ujjwalredd/sarathi`](https://github.com/ujjwalredd/sarathi) | [`08a51154a2f30af3eb4f6acb11115b9db912c5f8`](https://github.com/ujjwalredd/sarathi/commit/08a51154a2f30af3eb4f6acb11115b9db912c5f8) | `.` | `sarathi` | `DIRECT` | skill | MIT | `UNRUN` | -| [`sofus-nl/cc-plugin-codex`](https://github.com/sofus-nl/cc-plugin-codex) | [`cc5123f7fa18db9c38f838a9b70119e5a0a6847c`](https://github.com/sofus-nl/cc-plugin-codex/commit/cc5123f7fa18db9c38f838a9b70119e5a0a6847c) | `.` | `cc-plugin-codex` | `DIRECT` | skill, hook | Apache-2.0 + NOTICE | `UNRUN` | -| [`RMI/speedy-skills`](https://github.com/RMI/speedy-skills) | [`e983f800056a12b63fd60d5148538f98aaafe643`](https://github.com/RMI/speedy-skills/commit/e983f800056a12b63fd60d5148538f98aaafe643) | `.` | `example-minimal` | `DIRECT` | skill | MIT | `UNRUN` | -| [`roadrunner-tuff/roadrunner-admin-plugin`](https://github.com/roadrunner-tuff/roadrunner-admin-plugin) | [`8e130c07656c8f9db8bf5431332c9aed60a4b133`](https://github.com/roadrunner-tuff/roadrunner-admin-plugin/commit/8e130c07656c8f9db8bf5431332c9aed60a4b133) | `.` | `roadrunner-admin` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | +| Repository | Commit | Marketplace root | Plugin | Version | Preparation | Expected kinds | License | Strict result | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| [`bitrouter/bitrouter`](https://github.com/bitrouter/bitrouter) | [`678384888b73fc290ce4ce503a8a7f2a5cbf6da8`](https://github.com/bitrouter/bitrouter/commit/678384888b73fc290ce4ce503a8a7f2a5cbf6da8) | `.` | `bitrouter` | `0.1.0` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | +| [`Cassette-Editor/oh-my-cassette`](https://github.com/Cassette-Editor/oh-my-cassette) | [`cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32`](https://github.com/Cassette-Editor/oh-my-cassette/commit/cdad1fd2f62544b65a01ad00f74b19fe3ce4ca32) | `.` | `oh-my-cassette` | `0.4.14` | `STATIC_ADAPTER:local-source-v1` | skill, MCP | MIT | `UNRUN` | +| [`mostlyharmless-ai/watercooler`](https://github.com/mostlyharmless-ai/watercooler) | [`a5efa89df02e7796e20881fef4847f129d84d367`](https://github.com/mostlyharmless-ai/watercooler/commit/a5efa89df02e7796e20881fef4847f129d84d367) | `.` | `watercooler` | `0.5.6` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | +| [`commercetools/commercetools-ai-plugins`](https://github.com/commercetools/commercetools-ai-plugins) | [`440d6bd56eb2969b6a0dd41e3fcff286def0787c`](https://github.com/commercetools/commercetools-ai-plugins/commit/440d6bd56eb2969b6a0dd41e3fcff286def0787c) | `.` | `commercetools` | `0.14.0` | `DIRECT` | skill, MCP | CC-BY-4.0 | `UNRUN` | +| [`agentis-tools/ctx`](https://github.com/agentis-tools/ctx) | [`1782436e0ebf8d95ef4c086d94351698c464c4ee`](https://github.com/agentis-tools/ctx/commit/1782436e0ebf8d95ef4c086d94351698c464c4ee) | `plugins/codex/ctx` | `ctx` | `0.4.0` | `DIRECT` | skill, hook | Apache-2.0 OR MIT | `UNRUN` | +| [`agentmail-to/agentmail-plugins`](https://github.com/agentmail-to/agentmail-plugins) | [`134887caf9375229415e09c760ae31baa4cc1ec3`](https://github.com/agentmail-to/agentmail-plugins/commit/134887caf9375229415e09c760ae31baa4cc1ec3) | `.` | `agentmail` | `0.3.0` | `DIRECT` | skill, MCP | MIT | `UNRUN` | +| [`ujjwalredd/sarathi`](https://github.com/ujjwalredd/sarathi) | [`08a51154a2f30af3eb4f6acb11115b9db912c5f8`](https://github.com/ujjwalredd/sarathi/commit/08a51154a2f30af3eb4f6acb11115b9db912c5f8) | `.` | `sarathi` | `0.6.0` | `DIRECT` | skill | MIT | `UNRUN` | +| [`sofus-nl/cc-plugin-codex`](https://github.com/sofus-nl/cc-plugin-codex) | [`cc5123f7fa18db9c38f838a9b70119e5a0a6847c`](https://github.com/sofus-nl/cc-plugin-codex/commit/cc5123f7fa18db9c38f838a9b70119e5a0a6847c) | `.` | `cc-plugin-codex` | `0.1.1` | `DIRECT` | skill, hook | Apache-2.0 + NOTICE | `UNRUN` | +| [`RMI/speedy-skills`](https://github.com/RMI/speedy-skills) | [`e983f800056a12b63fd60d5148538f98aaafe643`](https://github.com/RMI/speedy-skills/commit/e983f800056a12b63fd60d5148538f98aaafe643) | `.` | `example-minimal` | `0.1.0` | `DIRECT` | skill | MIT | `UNRUN` | +| [`roadrunner-tuff/roadrunner-admin-plugin`](https://github.com/roadrunner-tuff/roadrunner-admin-plugin) | [`8e130c07656c8f9db8bf5431332c9aed60a4b133`](https://github.com/roadrunner-tuff/roadrunner-admin-plugin/commit/8e130c07656c8f9db8bf5431332c9aed60a4b133) | `.` | `roadrunner-admin` | `0.1.0` | `DIRECT` | skill, MCP | Apache-2.0 | `UNRUN` | `local-source-v1` may change only `/plugins/0/source` in `.agents/plugins/marketplace.json` to `{"source":"local","path":"./"}`. @@ -34,15 +34,32 @@ other fixture receives a rewrite. - The runner must extract immutable commit archives into owned temporary state with traversal and symlink escape checks. - It must never execute fixture code, scripts, builds, package managers, hooks, - MCP servers, apps, authentication flows, or models. + MCP servers, apps, authentication flows, or models. The public-fixture cells + make only install/list and declaration/discovery requests; they do not call a + capability runtime endpoint. - Every Codex probe must use the existing read-only, network-denied, host-state-denied strict boundary. - The retained artifact may contain source/tree/adapter hashes, relative receipt names, sanitized receipts, and a summary. It must contain no upstream source. +- On trusted GitHub Actions runs, the summary binds the exact main-branch + commit, event, run URL, run attempt, and intended artifact name. The assigned + artifact ID is reconciled after upload because GitHub creates it only then. - A truthful plugin receipt may be `FAIL`. A fetch, isolation, tool, identity, privacy, or ledger error must fail the matrix and cannot be relabeled as a compatibility result. +- Every prepared tree must match its audited checkout and marketplace hashes. + The marketplace manifest must resolve the named plugin to the exact audited + local subtree, whose plugin manifest must match the fixed name and version. + Every receipt must contain the exact audited capability keys and evidence + sources, not merely one capability of each expected kind. + +The strict public cells do not add fixture-specific execution sentinels to +third-party repositories. Their non-execution claim is therefore bounded to +the checker request path above and the earlier exact-version synthetic run, +whose hook and MCP sentinels remained absent. It is not evidence that arbitrary +third-party commands could never attempt a side effect if Codex changed loader +behavior. ## Non-certifying diagnostics @@ -59,8 +76,9 @@ satisfy the strict 10-repository gate. After the fixed runner was implemented, a complete local environment matrix ran all 20 repository/version cells against the prepared `0.147.0` and `0.146.1` binaries. All 20 returned exit `0` and `PASS` with exact -repository/plugin/marketplace/version identities. Every declared skill was -`DISCOVERED_EFFECTIVE`; the `ctx` and `cc-plugin-codex` hooks were +repository/plugin/marketplace/source-subtree/version identities. This was +rerun after the exact runtime root/version gates were added. Every declared +skill was `DISCOVERED_EFFECTIVE`; the `ctx` and `cc-plugin-codex` hooks were `DISCOVERED_UNTRUSTED`; every MCP declaration was `DECLARED_ONLY`. No app or unexpected capability kind appeared. The temporary checkouts and receipts were removed after reconciliation. @@ -70,21 +88,24 @@ network or personal host state. It is not counted in the strict result column. On 2026-08-10, the production preparation path fetched, parsed, extracted, license-checked, hashed and cleaned all ten immutable archives without running -Docker or any upstream code. A second root-agent run reproduced all ten archive -hashes: +Docker or any upstream code. A root-agent integration then fetched all ten +archives again, validated each exact marketplace source, plugin subtree, +manifest name and version, and reproduced the canonical tree hashes below. +Directory entries are ordered by their UTF-8 bytes, so the hash contract does +not depend on locale or ICU behavior: | Fixture | Archive SHA-256 | Extracted checkout SHA-256 | Probed marketplace SHA-256 | | --- | --- | --- | --- | -| `bitrouter` | `cd173128072bc769995a46baf1c9b19b0215214e8719a2f3779d4e3d52a69351` | `32e3cf5e9211dc28c55bd4bc2e7b794a73d9c961805aa1cc8436851ae17bff73` | `32e3cf5e9211dc28c55bd4bc2e7b794a73d9c961805aa1cc8436851ae17bff73` | -| `oh-my-cassette` | `3d64d09b7fae024d53616d4b04173e1a584f7982d3971fb94d0ea54cfed37287` | `86f6123019a98f6e59e0a0b3c0a41759aa52982f3af3242961b7be5cfbcae4d3` | `6218b332623998fb9cb9b116614d2465e485b376bdc930976bee556ab417e8e4` | -| `watercooler` | `aed68325b301e45b422f491865e7c3325c53d966dc89d3294deba45fa280ba3c` | `7f40c2886ade5da2325c94852db6062fa4a95c0b76ae868cf9aa035e28da5420` | `7f40c2886ade5da2325c94852db6062fa4a95c0b76ae868cf9aa035e28da5420` | -| `commercetools` | `7e4ac439b75a064a08a6e2a107d1ea8bb0221b974807670cea56c63a6cfd9094` | `8620f6238f94edeebca2eafa8fe193d0f7694841740d0c30b226fbfe6783eacf` | `8620f6238f94edeebca2eafa8fe193d0f7694841740d0c30b226fbfe6783eacf` | -| `ctx` | `5ef97584aacb6874cede5780ee47d137597a6ebd1fd3ecf04f0e97c79dfd8dfd` | `78bedfa1c634771af25f27db4205079b973fa05ee58a4f17442595a08d406f48` | `552f221e0c7c8ba59956829b14e5ee5b1a242c81d72d533e1f6d9b1a5f805c93` | -| `agentmail` | `ec595bfaf2e7201ade5e8c5902b424caa05948d1c6558e3f53be3468f02eca14` | `eac1dc131996059547801663bc7cd36c38221b0b28d9012c062c4af441e66800` | `eac1dc131996059547801663bc7cd36c38221b0b28d9012c062c4af441e66800` | -| `sarathi` | `f4577e9777d44111b1074460eba28987c80db2296f1983a018b391d99d42e7d8` | `7b45528f5816ed23191abf7aaae2670bd342989f6774007eb07937665bbcdc02` | `7b45528f5816ed23191abf7aaae2670bd342989f6774007eb07937665bbcdc02` | -| `cc-plugin-codex` | `57cff2045571f47a75231700f496f4aaaa4d780e1904721e5679a5d68f420cc9` | `562c1c772a33588dd7364ce11316bac0225c135bb84da91a4fdb619c32e3ba74` | `562c1c772a33588dd7364ce11316bac0225c135bb84da91a4fdb619c32e3ba74` | -| `speedy-skills` | `6734bb6adb707b623cbb22a9a8c12c571d7fbe51b768cc066055aa86da6e6da0` | `60459f76469df1ae420fab647a91b4413b79310d6166921a633877bd1e84fc9c` | `60459f76469df1ae420fab647a91b4413b79310d6166921a633877bd1e84fc9c` | -| `roadrunner-admin` | `a54ddc226d7f2af17d867b82dd83ff0dfa330abb9dc5e6f80b3cf0d639ad4650` | `1b057c8ac8031ea74a02a5ee46cd716f3f020df3779d37c32f3f47ede6bb0b5b` | `1b057c8ac8031ea74a02a5ee46cd716f3f020df3779d37c32f3f47ede6bb0b5b` | +| `bitrouter` | `cd173128072bc769995a46baf1c9b19b0215214e8719a2f3779d4e3d52a69351` | `ba604cb6d8313594bebbdbc899f930195e92f6c12fa14371764f7dec2e25d4c9` | `ba604cb6d8313594bebbdbc899f930195e92f6c12fa14371764f7dec2e25d4c9` | +| `oh-my-cassette` | `3d64d09b7fae024d53616d4b04173e1a584f7982d3971fb94d0ea54cfed37287` | `9dffb7f24db16606eeb44f7a23746073716069e63e4cf58a07631c02e1f57177` | `32c159545ca3626c13dfae8f1c833e456584df10c204060539475fd6c301b8e8` | +| `watercooler` | `aed68325b301e45b422f491865e7c3325c53d966dc89d3294deba45fa280ba3c` | `ad91f57b0605df94a1361aa67c3b35314f5357583aaef7f364b397da5a00ea19` | `ad91f57b0605df94a1361aa67c3b35314f5357583aaef7f364b397da5a00ea19` | +| `commercetools` | `7e4ac439b75a064a08a6e2a107d1ea8bb0221b974807670cea56c63a6cfd9094` | `749bea75c483aecfcc71dc04919a13e170953996eeba9dfdff16c4f5f49073c0` | `749bea75c483aecfcc71dc04919a13e170953996eeba9dfdff16c4f5f49073c0` | +| `ctx` | `5ef97584aacb6874cede5780ee47d137597a6ebd1fd3ecf04f0e97c79dfd8dfd` | `7f6934a57be05a126b968a5c5d346fb9d3150bdb6a57e199d2274204c94337eb` | `7b3212dbd512ee0bbf7f8c3c2b69c86bdba41ed55f9f71e5f69e63dc0cce49f7` | +| `agentmail` | `ec595bfaf2e7201ade5e8c5902b424caa05948d1c6558e3f53be3468f02eca14` | `97a82ec2aaa745663a6baa0dab16c476858d4ddc47230f2906b26eafbffa6669` | `97a82ec2aaa745663a6baa0dab16c476858d4ddc47230f2906b26eafbffa6669` | +| `sarathi` | `f4577e9777d44111b1074460eba28987c80db2296f1983a018b391d99d42e7d8` | `da617a7857381c86ef963f85185a1afef851369a52cb4630d9360c21df904599` | `da617a7857381c86ef963f85185a1afef851369a52cb4630d9360c21df904599` | +| `cc-plugin-codex` | `57cff2045571f47a75231700f496f4aaaa4d780e1904721e5679a5d68f420cc9` | `1da66cadeb01bb4b57f63e522fdc763df98d5ba6e2034e04599b02c1e394daed` | `1da66cadeb01bb4b57f63e522fdc763df98d5ba6e2034e04599b02c1e394daed` | +| `speedy-skills` | `6734bb6adb707b623cbb22a9a8c12c571d7fbe51b768cc066055aa86da6e6da0` | `d39797da9765bf1d822887dc6735f186d4bfc199279558817cf6eef375aab1c2` | `d39797da9765bf1d822887dc6735f186d4bfc199279558817cf6eef375aab1c2` | +| `roadrunner-admin` | `a54ddc226d7f2af17d867b82dd83ff0dfa330abb9dc5e6f80b3cf0d639ad4650` | `33eaa8e5aef5449d496771317d1f40205dabf06fa6d6041e1e142e61e651f5d1` | `33eaa8e5aef5449d496771317d1f40205dabf06fa6d6041e1e142e61e651f5d1` | For `oh-my-cassette`, the observed adapter input is `d5c629f3a3b8dd2cdf560963c26b1c5e9dc062045fef13cafca178e7b1bd3d3f` @@ -96,7 +117,11 @@ preparation receipts, not Codex compatibility receipts. ## Publication gate Change this decision from `HOLD` only after all 20 strict cells are retained, -validated against their exact repository/plugin/version identities, scanned -for private paths and execution sentinels, and independently reconciled with -the immutable inputs above. Stars, a green workflow, or a partial matrix do not -substitute for those receipts. +validated against their exact repository/tree/plugin/version/capability +identities, scanned for private paths, and independently reconciled with the +immutable inputs above. A code-`1` receipt remains plain `FAIL`/`HOLD`; it does +not establish a version or API incompatibility without separate causal +evidence. The private workflow retains its artifact for 90 days. Before a +public release, the sanitized receipts and summary must also become durable +release evidence rather than relying on an expiring Actions URL. Stars, a green +workflow, or a partial matrix do not substitute for those receipts. diff --git a/docs/evidence/technical-falsifier.md b/docs/evidence/technical-falsifier.md index 4f24b58..d287726 100644 --- a/docs/evidence/technical-falsifier.md +++ b/docs/evidence/technical-falsifier.md @@ -115,7 +115,7 @@ strict gate. explicit reason `strict released integration requires CODEX_RELEASED_FALSIFIER_OPT_IN=1`. -The complete `npm test` suite observed 122 passing tests, zero failures, and +The complete `npm test` suite observed 158 passing tests, zero failures, and the same one strict Linux test skipped. With both exact version variables set, `npm run falsify` exited `1` with: diff --git a/docs/superpowers/plans/2026-08-10-codex-plugin-check.md b/docs/superpowers/plans/2026-08-10-codex-plugin-check.md index 4d0d3f5..7da7748 100644 --- a/docs/superpowers/plans/2026-08-10-codex-plugin-check.md +++ b/docs/superpowers/plans/2026-08-10-codex-plugin-check.md @@ -496,8 +496,9 @@ git commit -m "docs: prepare evidence-bound v0 release" Bind exactly ten unique HTTPS GitHub repositories, full commit SHAs, expected plugin IDs, marketplace roots, licenses, and adapter IDs. Reject mutable refs, path escapes, duplicate rows, extra adapter fields, untrusted workflow events, -receipt identity drift, missing expected capability kinds, absolute evidence -paths, and arbitrary tool errors mislabeled as incompatibility. +receipt identity drift, any missing or substituted capability key/source, +prepared tree or manifest/version drift, absolute evidence paths, and arbitrary +tool errors or plain conformance failures mislabeled as incompatibility. - [x] **Step 2: Verify RED** @@ -532,8 +533,10 @@ hashes, sanitized receipts, run SHA/URL, and artifact identity. Independently compare the artifact to immutable source expectations. Mark the technical public-fixture gate `PASS` only when all ten fixtures have complete -observations or an evidence-backed version/API incompatibility classification. -Keep project/publication `HOLD` for any unexplained error or missing row. +observations. A failed conformance receipt stays `FAIL`/`HOLD` unless separate +evidence establishes its cause; do not infer a version/API incompatibility from +exit code `1`. Keep project/publication `HOLD` for any unexplained error or +missing row. ### Task 8: Bounded maintainer validation and application hold diff --git a/scripts/falsify-public-fixtures.mjs b/scripts/falsify-public-fixtures.mjs index 34fda67..f27b01e 100644 --- a/scripts/falsify-public-fixtures.mjs +++ b/scripts/falsify-public-fixtures.mjs @@ -196,11 +196,175 @@ const FIXED_FIXTURES = [ } ]; +function fixedCapabilities(rows) { + return rows.map(([kind, key, source]) => ({ kind, key, source })); +} + +const FIXED_PLUGIN_CONTRACTS = { + bitrouter: { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: '.', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.1.0', + expectedCheckoutSha256: 'ba604cb6d8313594bebbdbc899f930195e92f6c12fa14371764f7dec2e25d4c9', + expectedMarketplaceSha256: 'ba604cb6d8313594bebbdbc899f930195e92f6c12fa14371764f7dec2e25d4c9', + expectedCapabilities: fixedCapabilities([ + ['mcp', 'bitrouter', 'plugin/read'], + ['skill', 'bitrouter:bitrouter', 'plugin/read + skills/list'] + ]) + }, + 'oh-my-cassette': { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: '.', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.4.14', + expectedCheckoutSha256: '9dffb7f24db16606eeb44f7a23746073716069e63e4cf58a07631c02e1f57177', + expectedMarketplaceSha256: '32c159545ca3626c13dfae8f1c833e456584df10c204060539475fd6c301b8e8', + expectedCapabilities: fixedCapabilities([ + ['mcp', 'cassette', 'plugin/read'], + ['skill', 'oh-my-cassette:cassette-model', 'plugin/read + skills/list'], + ['skill', 'oh-my-cassette:cassette-video-edit', 'plugin/read + skills/list'] + ]) + }, + watercooler: { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: 'plugins/codex/watercooler', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.5.6', + expectedCheckoutSha256: 'ad91f57b0605df94a1361aa67c3b35314f5357583aaef7f364b397da5a00ea19', + expectedMarketplaceSha256: 'ad91f57b0605df94a1361aa67c3b35314f5357583aaef7f364b397da5a00ea19', + expectedCapabilities: fixedCapabilities([ + ['mcp', 'watercooler', 'plugin/read'], + ['skill', 'watercooler:find-related', 'plugin/read + skills/list'], + ['skill', 'watercooler:recall', 'plugin/read + skills/list'], + ['skill', 'watercooler:search-threads', 'plugin/read + skills/list'], + ['skill', 'watercooler:threads', 'plugin/read + skills/list'], + ['skill', 'watercooler:update-agent-context', 'plugin/read + skills/list'], + ['skill', 'watercooler:watercooler-health', 'plugin/read + skills/list'], + ['skill', 'watercooler:watercooler-onboarding', 'plugin/read + skills/list'] + ]) + }, + commercetools: { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: '.agents/plugins/commercetools', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.14.0', + expectedCheckoutSha256: '749bea75c483aecfcc71dc04919a13e170953996eeba9dfdff16c4f5f49073c0', + expectedMarketplaceSha256: '749bea75c483aecfcc71dc04919a13e170953996eeba9dfdff16c4f5f49073c0', + expectedCapabilities: fixedCapabilities([ + ['mcp', 'commerce-mcp', 'plugin/read'], + ['mcp', 'commercetools-knowledge', 'plugin/read'], + ['skill', 'commercetools:commercetools-checkout', 'plugin/read + skills/list'], + ['skill', 'commercetools:commercetools-commerce-patterns', 'plugin/read + skills/list'], + ['skill', 'commercetools:commercetools-connect', 'plugin/read + skills/list'], + ['skill', 'commercetools:commercetools-platform', 'plugin/read + skills/list'], + ['skill', 'commercetools:commercetools-storefront', 'plugin/read + skills/list'] + ]) + }, + ctx: { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: '.', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.4.0', + expectedCheckoutSha256: '7f6934a57be05a126b968a5c5d346fb9d3150bdb6a57e199d2274204c94337eb', + expectedMarketplaceSha256: '7b3212dbd512ee0bbf7f8c3c2b69c86bdba41ed55f9f71e5f69e63dc0cce49f7', + expectedCapabilities: fixedCapabilities([ + ['hook', 'ctx@ctx-local:hooks/hooks.json:post_tool_use:0:0', 'plugin/read + hooks/list'], + ['hook', 'ctx@ctx-local:hooks/hooks.json:session_start:0:0', 'plugin/read + hooks/list'], + ['hook', 'ctx@ctx-local:hooks/hooks.json:stop:0:0', 'plugin/read + hooks/list'], + ['skill', 'ctx:ctx', 'plugin/read + skills/list'] + ]) + }, + agentmail: { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: '.', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.3.0', + expectedCheckoutSha256: '97a82ec2aaa745663a6baa0dab16c476858d4ddc47230f2906b26eafbffa6669', + expectedMarketplaceSha256: '97a82ec2aaa745663a6baa0dab16c476858d4ddc47230f2906b26eafbffa6669', + expectedCapabilities: fixedCapabilities([ + ['mcp', 'agentmail', 'plugin/read'], + ['skill', 'agentmail:agent-email-patterns', 'plugin/read + skills/list'], + ['skill', 'agentmail:agentmail', 'plugin/read + skills/list'], + ['skill', 'agentmail:agentmail-cli', 'plugin/read + skills/list'], + ['skill', 'agentmail:agentmail-mcp', 'plugin/read + skills/list'], + ['skill', 'agentmail:agentmail-toolkit', 'plugin/read + skills/list'], + ['skill', 'agentmail:check-email', 'plugin/read + skills/list'], + ['skill', 'agentmail:manage-inboxes', 'plugin/read + skills/list'], + ['skill', 'agentmail:send-email', 'plugin/read + skills/list'] + ]) + }, + sarathi: { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: '.', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.6.0', + expectedCheckoutSha256: 'da617a7857381c86ef963f85185a1afef851369a52cb4630d9360c21df904599', + expectedMarketplaceSha256: 'da617a7857381c86ef963f85185a1afef851369a52cb4630d9360c21df904599', + expectedCapabilities: fixedCapabilities([ + ['skill', 'sarathi:sarathi', 'plugin/read + skills/list'] + ]) + }, + 'cc-plugin-codex': { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: 'plugins/cc-plugin-codex', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.1.1', + expectedCheckoutSha256: '1da66cadeb01bb4b57f63e522fdc763df98d5ba6e2034e04599b02c1e394daed', + expectedMarketplaceSha256: '1da66cadeb01bb4b57f63e522fdc763df98d5ba6e2034e04599b02c1e394daed', + expectedCapabilities: fixedCapabilities([ + ['hook', 'cc-plugin-codex@cc-plugin-codex:hooks/hooks.json:session_end:0:0', 'plugin/read + hooks/list'], + ['hook', 'cc-plugin-codex@cc-plugin-codex:hooks/hooks.json:stop:0:0', 'plugin/read + hooks/list'], + ['skill', 'cc-plugin-codex:claude-adversarial-review', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-cancel', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-cli-runtime', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-prompting', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-rescue', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-result', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-result-handling', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-review', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-setup', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-status', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-transfer', 'plugin/read + skills/list'] + ]) + }, + 'speedy-skills': { + marketplaceManifestPath: '.claude-plugin/marketplace.json', + pluginRoot: 'plugins/example-minimal', + pluginManifestPath: '.claude-plugin/plugin.json', + pluginVersion: '0.1.0', + expectedCheckoutSha256: 'd39797da9765bf1d822887dc6735f186d4bfc199279558817cf6eef375aab1c2', + expectedMarketplaceSha256: 'd39797da9765bf1d822887dc6735f186d4bfc199279558817cf6eef375aab1c2', + expectedCapabilities: fixedCapabilities([ + ['skill', 'example-minimal:summarizing-git-log', 'plugin/read + skills/list'] + ]) + }, + 'roadrunner-admin': { + marketplaceManifestPath: '.agents/plugins/marketplace.json', + pluginRoot: 'plugins/roadrunner-admin', + pluginManifestPath: '.codex-plugin/plugin.json', + pluginVersion: '0.1.0', + expectedCheckoutSha256: '33eaa8e5aef5449d496771317d1f40205dabf06fa6d6041e1e142e61e651f5d1', + expectedMarketplaceSha256: '33eaa8e5aef5449d496771317d1f40205dabf06fa6d6041e1e142e61e651f5d1', + expectedCapabilities: fixedCapabilities([ + ['mcp', 'roadrunner-admin', 'plugin/read'], + ['skill', 'roadrunner-admin:roadrunner-admin', 'plugin/read + skills/list'] + ]) + } +}; + +for (const fixture of FIXED_FIXTURES) { + Object.assign(fixture, FIXED_PLUGIN_CONTRACTS[fixture.evidenceId]); +} + export const PUBLIC_FIXTURES = Object.freeze( FIXED_FIXTURES.map((fixture) => Object.freeze({ ...fixture, licensePaths: Object.freeze([...fixture.licensePaths]), - expectedKinds: Object.freeze([...fixture.expectedKinds]) + expectedKinds: Object.freeze([...fixture.expectedKinds]), + expectedCapabilities: Object.freeze(fixture.expectedCapabilities.map((capability) => ( + Object.freeze({ ...capability }) + ))) })) ); @@ -232,6 +396,49 @@ export function validateFixtureDefinitions(fixtures) { return fixtures; } +function githubProvenanceFromEnvironment(env) { + const keys = [ + 'GITHUB_ACTIONS', + 'GITHUB_EVENT_NAME', + 'GITHUB_REF', + 'GITHUB_REPOSITORY', + 'GITHUB_RUN_ATTEMPT', + 'GITHUB_RUN_ID', + 'GITHUB_SERVER_URL', + 'GITHUB_SHA' + ]; + const hasGithubValue = keys.some((key) => ( + typeof env[key] === 'string' && env[key] !== '' + )); + if (!hasGithubValue) return null; + if ( + env.GITHUB_ACTIONS !== 'true' || + !['push', 'workflow_dispatch'].includes(env.GITHUB_EVENT_NAME) || + env.GITHUB_REF !== 'refs/heads/main' || + env.GITHUB_REPOSITORY !== 'builtbyhuy/codex-plugin-check' || + !/^[1-9]\d*$/.test(env.GITHUB_RUN_ATTEMPT ?? '') || + !/^[1-9]\d*$/.test(env.GITHUB_RUN_ID ?? '') || + env.GITHUB_SERVER_URL !== 'https://github.com' || + !COMMIT_SHA.test(env.GITHUB_SHA ?? '') + ) { + throw new Error('GitHub Actions provenance was incomplete or outside trusted main'); + } + const runUrl = `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${ + env.GITHUB_RUN_ID + }`; + return { + provider: 'github-actions', + repository: env.GITHUB_REPOSITORY, + commit: env.GITHUB_SHA, + ref: env.GITHUB_REF, + event: env.GITHUB_EVENT_NAME, + runId: env.GITHUB_RUN_ID, + runAttempt: env.GITHUB_RUN_ATTEMPT, + runUrl, + artifactName: `public-fixture-evidence-${env.GITHUB_RUN_ID}-${env.GITHUB_RUN_ATTEMPT}` + }; +} + export function publicFixtureOptionsFromEnvironment(env = process.env, cwd = process.cwd()) { if ( env.CODEX_CURRENT_VERSION !== CODEX_VERSIONS[0] || @@ -261,7 +468,8 @@ export function publicFixtureOptionsFromEnvironment(env = process.env, cwd = pro architecture: process.arch, outputBoundary, outputRoot, - platform: process.platform + platform: process.platform, + provenance: githubProvenanceFromEnvironment(env) }; } @@ -707,7 +915,10 @@ async function hashDirectory(root) { const hash = createHash('sha256'); async function visit(directory, relativeDirectory = '') { const entries = await readdir(directory, { withFileTypes: true }); - entries.sort((left, right) => left.name.localeCompare(right.name, 'en')); + entries.sort((left, right) => Buffer.compare( + Buffer.from(left.name, 'utf8'), + Buffer.from(right.name, 'utf8') + )); for (const entry of entries) { const relative = relativeDirectory === '' ? entry.name @@ -740,9 +951,120 @@ async function assertRegularFixtureFile(checkoutRoot, relative, label) { if (metadata.isSymbolicLink() || !metadata.isFile()) { throw new Error(`${label} must be a regular file, not a symlink`); } + if (await realpath(absolute) !== absolute) { + throw new Error(`${label} must not traverse a symlink`); + } return absolute; } +function parseManifestObject(bytes, label) { + let parsed; + try { + parsed = JSON.parse(bytes); + } catch (cause) { + throw new Error(`${label} was not valid JSON`, { cause }); + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`${label} must be a JSON object`); + } + return parsed; +} + +function localPluginSourcePath(source) { + if (typeof source === 'string') return source; + if ( + source !== null && + typeof source === 'object' && + !Array.isArray(source) && + source.source === 'local' && + typeof source.path === 'string' + ) { + return source.path; + } + throw new Error('Marketplace plugin source must be a fixed local path'); +} + +export async function validateStaticPluginContract(fixture, marketplaceRoot) { + assertFixedFixture(fixture); + const canonicalMarketplaceRoot = await realpath(marketplaceRoot); + if (canonicalMarketplaceRoot !== path.resolve(marketplaceRoot)) { + throw new Error('Marketplace root must not traverse a symlink'); + } + const marketplaceManifestFile = await assertRegularFixtureFile( + canonicalMarketplaceRoot, + fixture.marketplaceManifestPath, + 'Marketplace manifest' + ); + const marketplaceManifest = parseManifestObject( + await readFile(marketplaceManifestFile, 'utf8'), + 'Marketplace manifest' + ); + if (marketplaceManifest.name !== fixture.marketplace) { + throw new Error('Marketplace manifest returned the wrong marketplace name'); + } + if (!Array.isArray(marketplaceManifest.plugins)) { + throw new Error('Marketplace manifest must contain a plugin list'); + } + const pluginEntries = marketplaceManifest.plugins.filter( + (entry) => entry !== null && typeof entry === 'object' && entry.name === fixture.plugin + ); + if (pluginEntries.length !== 1) { + throw new Error('Marketplace manifest must contain exactly one expected plugin entry'); + } + const sourcePath = localPluginSourcePath(pluginEntries[0].source); + if ( + sourcePath === '' || + path.isAbsolute(sourcePath) || + sourcePath.includes('\\') || + sourcePath.includes('\0') + ) { + throw new Error('Marketplace plugin source path must be a safe relative path'); + } + const requestedPluginRoot = path.resolve(canonicalMarketplaceRoot, sourcePath); + const expectedPluginRoot = path.resolve( + canonicalMarketplaceRoot, + ...fixture.pluginRoot.split('/') + ); + if ( + requestedPluginRoot !== expectedPluginRoot || + !pathIsWithin(requestedPluginRoot, canonicalMarketplaceRoot) + ) { + throw new Error('Marketplace plugin source did not match the audited plugin root'); + } + const canonicalPluginRoot = await realpath(requestedPluginRoot); + const pluginRootMetadata = await lstat(canonicalPluginRoot); + if ( + canonicalPluginRoot !== requestedPluginRoot || + !pluginRootMetadata.isDirectory() || + !pathIsWithin(canonicalPluginRoot, canonicalMarketplaceRoot) + ) { + throw new Error('Audited plugin root must be a real directory inside the marketplace'); + } + const pluginManifestFile = await assertRegularFixtureFile( + canonicalPluginRoot, + fixture.pluginManifestPath, + 'Plugin manifest' + ); + const pluginManifest = parseManifestObject( + await readFile(pluginManifestFile, 'utf8'), + 'Plugin manifest' + ); + if (pluginManifest.name !== fixture.plugin) { + throw new Error('Plugin manifest returned the wrong plugin name'); + } + if (pluginManifest.version !== fixture.pluginVersion) { + throw new Error('Plugin manifest returned the wrong plugin version'); + } + return { + marketplaceManifestPath: fixture.marketplaceManifestPath, + marketplace: fixture.marketplace, + plugin: fixture.plugin, + pluginRoot: fixture.pluginRoot, + pluginManifestPath: fixture.pluginManifestPath, + pluginVersion: fixture.pluginVersion + }; +} + export async function preparePublicFixture({ fixture, temporaryRoot }, dependencies = {}) { assertFixedFixture(fixture); const canonicalTemporaryRoot = await realpath(temporaryRoot); @@ -798,12 +1120,14 @@ export async function preparePublicFixture({ fixture, temporaryRoot }, dependenc throw new Error('Marketplace root must be an owned fixture directory'); } await auditExtractedSymlinks(checkoutRoot); + const pluginContract = await validateStaticPluginContract(fixture, marketplaceRoot); return { archiveSha256, checkoutSha256, marketplaceSha256: await hashDirectory(marketplaceRoot), marketplaceRoot, - adapter + adapter, + pluginContract }; } @@ -841,19 +1165,46 @@ export function validatePublicReceipt(receipt, options) { if (receipt.plugin.marketplace !== fixture.marketplace) { throw new Error(`Public fixture ${fixture.repository} returned the wrong marketplace identity`); } - const expectedKinds = new Set(fixture.expectedKinds); - const observedKinds = new Set(); + const markers = options.personalMarkers ?? []; + const values = allStringValues(receipt); + for (const marker of markers) { + if (typeof marker === 'string' && marker !== '' && values.some((value) => value.includes(marker))) { + throw new Error(`Public fixture ${fixture.repository} receipt leaked a personal host path`); + } + } + for (const value of values) { + if ( + value !== '/workspace' && + (/^(?:\/Users\/|\/home\/)/.test(value) || /^[A-Za-z]:[\\/]/.test(value)) + ) { + throw new Error(`Public fixture ${fixture.repository} receipt leaked an absolute host path`); + } + } + const expectedCapabilities = new Map( + fixture.expectedCapabilities.map((capability) => [ + `${capability.kind}\0${capability.key}`, + capability + ]) + ); + if (receipt.capabilities.length !== expectedCapabilities.size) { + throw new Error( + `Public fixture ${fixture.repository} returned the wrong capability count` + ); + } const capabilityKeys = new Set(); for (const capability of receipt.capabilities) { - if (!expectedKinds.has(capability.kind)) { - throw new Error(`Public fixture ${fixture.repository} returned an unexpected capability kind`); - } const identity = `${capability.kind}\0${capability.key}`; if (capabilityKeys.has(identity)) { throw new Error(`Public fixture ${fixture.repository} returned a duplicate capability`); } + const expectedCapability = expectedCapabilities.get(identity); + if (expectedCapability === undefined) { + throw new Error(`Public fixture ${fixture.repository} returned an unexpected capability key`); + } + if (capability.source !== expectedCapability.source) { + throw new Error(`Public fixture ${fixture.repository} returned the wrong capability source`); + } capabilityKeys.add(identity); - observedKinds.add(capability.kind); if ( capability.kind === 'skill' && !['DISCOVERED_EFFECTIVE', ...(expectedExitCode === 1 ? ['MISSING'] : [])] @@ -879,24 +1230,9 @@ export function validatePublicReceipt(receipt, options) { throw new Error(`Public fixture ${fixture.repository} MCP capability made a runtime claim`); } } - for (const kind of expectedKinds) { - if (!observedKinds.has(kind)) { - throw new Error(`Public fixture ${fixture.repository} is missing capability kind ${kind}`); - } - } - const markers = options.personalMarkers ?? []; - const values = allStringValues(receipt); - for (const marker of markers) { - if (typeof marker === 'string' && marker !== '' && values.some((value) => value.includes(marker))) { - throw new Error(`Public fixture ${fixture.repository} receipt leaked a personal host path`); - } - } - for (const value of values) { - if ( - value !== '/workspace' && - (/^(?:\/Users\/|\/home\/)/.test(value) || /^[A-Za-z]:[\\/]/.test(value)) - ) { - throw new Error(`Public fixture ${fixture.repository} receipt leaked an absolute host path`); + for (const identity of expectedCapabilities.keys()) { + if (!capabilityKeys.has(identity)) { + throw new Error(`Public fixture ${fixture.repository} is missing an expected capability`); } } return receipt; @@ -1101,6 +1437,8 @@ export async function probePublicFixtureCell(options, dependencies = {}) { const code = await cliMain([ '--marketplace-root', marketplaceRoot, '--plugin', options.fixture.plugin, + '--expected-plugin-root', options.fixture.pluginRoot, + '--expected-plugin-version', options.fixture.pluginVersion, '--codex-version', options.version, '--cwd', marketplaceRoot, '--output', stagedPath, @@ -1139,9 +1477,20 @@ export async function probePublicFixtureCell(options, dependencies = {}) { } } -function inspectDockerReal() { +export function inspectDockerReal(dependencies = {}) { return new Promise((resolve, reject) => { - const child = spawn('docker', ['version', '--format', '{{.Server.Version}}'], { + const spawnProcess = dependencies.spawnProcess ?? spawn; + const timeoutMs = dependencies.timeoutMs ?? DOCKER_TIMEOUT_MS; + const killGraceMs = dependencies.killGraceMs ?? 100; + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + reject(new Error('Docker inspection timeout must be a positive integer')); + return; + } + if (!Number.isSafeInteger(killGraceMs) || killGraceMs <= 0) { + reject(new Error('Docker inspection kill grace must be a positive integer')); + return; + } + const child = spawnProcess('docker', ['version', '--format', '{{.Server.Version}}'], { shell: false, stdio: ['ignore', 'pipe', 'pipe'] }); @@ -1149,10 +1498,14 @@ function inspectDockerReal() { const stderr = []; let outputBytes = 0; let forcedError; + let killTimer; let settled = false; const terminate = (cause) => { forcedError ??= cause; + if (killTimer !== undefined) return; child.kill('SIGTERM'); + killTimer = setTimeout(() => child.kill('SIGKILL'), killGraceMs); + killTimer.unref?.(); }; for (const [stream, chunks] of [[child.stdout, stdout], [child.stderr, stderr]]) { stream.on('data', (chunk) => { @@ -1165,20 +1518,22 @@ function inspectDockerReal() { }); } const timer = setTimeout( - () => terminate(new Error(`Docker inspection timed out after ${DOCKER_TIMEOUT_MS}ms`)), - DOCKER_TIMEOUT_MS + () => terminate(new Error(`Docker inspection timed out after ${timeoutMs}ms`)), + timeoutMs ); timer.unref?.(); child.once('error', (cause) => { if (settled) return; settled = true; clearTimeout(timer); + clearTimeout(killTimer); reject(cause); }); child.once('close', (code) => { if (settled) return; settled = true; clearTimeout(timer); + clearTimeout(killTimer); if (forcedError) { reject(forcedError); return; @@ -1241,6 +1596,12 @@ function validatePreparedFixture(prepared, fixture, temporaryRoot) { throw new Error(`Prepared public fixture is missing ${field}`); } } + if (prepared.checkoutSha256 !== fixture.expectedCheckoutSha256) { + throw new Error('Prepared public fixture checkout hash did not match the audited tree'); + } + if (prepared.marketplaceSha256 !== fixture.expectedMarketplaceSha256) { + throw new Error('Prepared public fixture marketplace hash did not match the audited tree'); + } if (fixture.adapterId === 'none') { if (prepared.adapter !== null) throw new Error('DIRECT fixture must not record an adapter'); } else if ( @@ -1253,6 +1614,25 @@ function validatePreparedFixture(prepared, fixture, temporaryRoot) { if (!pathIsWithin(prepared.marketplaceRoot, temporaryRoot)) { throw new Error('Prepared marketplace root escaped owned temporary state'); } + const expectedContract = { + marketplaceManifestPath: fixture.marketplaceManifestPath, + marketplace: fixture.marketplace, + plugin: fixture.plugin, + pluginRoot: fixture.pluginRoot, + pluginManifestPath: fixture.pluginManifestPath, + pluginVersion: fixture.pluginVersion + }; + if ( + prepared.pluginContract === null || + typeof prepared.pluginContract !== 'object' || + Array.isArray(prepared.pluginContract) || + Object.keys(prepared.pluginContract).length !== Object.keys(expectedContract).length || + Object.entries(expectedContract).some( + ([key, value]) => prepared.pluginContract[key] !== value + ) + ) { + throw new Error('Prepared public fixture plugin contract did not match the audited identity'); + } return prepared; } @@ -1265,11 +1645,20 @@ function summaryFixture(fixture, prepared, receipts) { classification: fixture.classification, adapterId: fixture.adapterId, marketplaceRoot: fixture.marketplaceRoot, + marketplaceManifestPath: fixture.marketplaceManifestPath, plugin: fixture.plugin, + pluginRoot: fixture.pluginRoot, + pluginManifestPath: fixture.pluginManifestPath, + pluginVersion: fixture.pluginVersion, marketplace: fixture.marketplace, license: fixture.license, licensePaths: [...fixture.licensePaths], expectedKinds: [...fixture.expectedKinds], + expectedCapabilities: fixture.expectedCapabilities.map(({ kind, key, source }) => ({ + kind, + key, + source + })), archiveUrl: fixture.archiveUrl, archiveSha256: prepared.archiveSha256, checkoutSha256: prepared.checkoutSha256, @@ -1299,8 +1688,8 @@ export async function runPublicFixtureMatrix(options, dependencies = {}) { ownedOutput = await createOwnedOutputRoot(options.outputRoot, options.outputBoundary); temporaryRoot = await realpath(await makeTemporaryRoot()); const fixturesSummary = []; - let compatibleCells = 0; - let incompatibleCells = 0; + let passedCells = 0; + let failedCells = 0; for (const fixture of fixtures) { const prepared = validatePreparedFixture( await prepareFixture({ fixture, temporaryRoot }), @@ -1334,9 +1723,9 @@ export async function runPublicFixtureMatrix(options, dependencies = {}) { })); await assertOwnedOutputRoot(ownedOutput.outputRoot, ownedOutput.boundary); await writeAtomicEvidenceJson(ownedOutput, evidencePath, receipt); - const outcome = staged.code === 0 ? 'PASS' : 'VERSION_OR_API_INCOMPATIBLE'; - if (outcome === 'PASS') compatibleCells += 1; - else incompatibleCells += 1; + const outcome = staged.code === 0 ? 'PASS' : 'FAIL'; + if (outcome === 'PASS') passedCells += 1; + else failedCells += 1; receipts.push({ version, receiptPath: evidencePath, outcome }); } fixturesSummary.push(summaryFixture(fixture, prepared, receipts)); @@ -1345,16 +1734,17 @@ export async function runPublicFixtureMatrix(options, dependencies = {}) { temporaryRoot = undefined; const summary = { schemaVersion: '0.1.0', - status: incompatibleCells === 0 ? 'PASS' : 'HOLD', + status: failedCells === 0 ? 'PASS' : 'HOLD', outputRoot: '.', platform: `linux-${options.architecture}`, + provenance: options.provenance ?? null, versions: [...versions], gate: { observed: fixtures.length, required: 10, cells: fixtures.length * versions.length, - compatibleCells, - incompatibleCells + passedCells, + failedCells }, fixtures: fixturesSummary }; diff --git a/src/check-plugin.mjs b/src/check-plugin.mjs index 1fa18b4..d5e2f2b 100644 --- a/src/check-plugin.mjs +++ b/src/check-plugin.mjs @@ -5,6 +5,8 @@ import { AppServerClient } from './app-server-client.mjs'; import { createIsolation as createRealIsolation } from './isolation.mjs'; import { buildReceipt } from './receipt.mjs'; +const EXACT_PLUGIN_VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/; + function runRealCommand({ command, args, cwd, env }) { return new Promise((resolve, reject) => { const child = spawn(command, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }); @@ -42,7 +44,7 @@ function marketplaceFrom(value, marketplaceRoot) { return value; } -function installedPluginFrom(list, install, marketplaceRoot) { +function installedPluginFrom(list, install, marketplaceRoot, expectedPlugin) { const installed = Array.isArray(list?.installed) ? list.installed.find((item) => item?.pluginId === install.pluginId) : undefined; @@ -52,6 +54,8 @@ function installedPluginFrom(list, install, marketplaceRoot) { installed.installed === true && installed.enabled === true && installed.source?.source === 'local' && pathIsWithin(installed.source.path, marketplaceRoot) && + (expectedPlugin === null || + path.resolve(installed.source.path) === expectedPlugin.root) && installed.marketplaceSource?.sourceType === 'local' && path.resolve(installed.marketplaceSource.source ?? '') === path.resolve(marketplaceRoot); if (!matches) { @@ -60,6 +64,27 @@ function installedPluginFrom(list, install, marketplaceRoot) { return installed; } +async function expectedPluginFrom(options, marketplaceRoot) { + const hasRoot = options.expectedPluginRoot !== undefined; + const hasVersion = options.expectedPluginVersion !== undefined; + if (!hasRoot && !hasVersion) return null; + if ( + !hasRoot || + !hasVersion || + typeof options.expectedPluginRoot !== 'string' || + options.expectedPluginRoot === '' || + typeof options.expectedPluginVersion !== 'string' || + !EXACT_PLUGIN_VERSION.test(options.expectedPluginVersion) + ) { + throw new Error('Expected plugin root and version must be supplied together'); + } + const root = await realpath(path.resolve(options.expectedPluginRoot)); + if (!pathIsWithin(root, marketplaceRoot)) { + throw new Error('Expected plugin root escaped the requested marketplace'); + } + return { root, version: options.expectedPluginVersion }; +} + function pathIsWithin(candidate, root) { if (typeof candidate !== 'string' || typeof root !== 'string') return false; const relative = path.relative(path.resolve(root), path.resolve(candidate)); @@ -133,6 +158,7 @@ export async function checkPlugin(options, dependencies = {}) { const startAppServer = dependencies.startAppServer ?? AppServerClient.start; const makeIsolation = dependencies.createIsolation ?? createRealIsolation; const marketplaceRoot = await realpath(path.resolve(options.marketplaceRoot)); + const expectedPlugin = await expectedPluginFrom(options, marketplaceRoot); const isolation = await makeIsolation({ targetRoot: marketplaceRoot, receiptPath: options.output ?? 'conformance.json', @@ -169,8 +195,11 @@ export async function checkPlugin(options, dependencies = {}) { install.marketplaceName !== marketplace.marketplaceName) { throw new Error('Codex plugin add returned invalid install JSON'); } + if (expectedPlugin !== null && install.version !== expectedPlugin.version) { + throw new Error('Codex installed a different plugin version than expected'); + } const list = JSON.parse((await runCommand(invocation(['plugin', 'list', '--json']))).stdout); - const installed = installedPluginFrom(list, install, marketplaceRoot); + const installed = installedPluginFrom(list, install, marketplaceRoot, expectedPlugin); client = await startAppServer({ command: options.codex ?? 'codex', args: ['app-server', '--stdio', '--disable', 'remote_plugin'], diff --git a/src/cli.mjs b/src/cli.mjs index 8bbcf25..5e6a200 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -21,6 +21,8 @@ import { exitCodeForStatus } from './receipt.mjs'; const VALUE_FLAGS = new Set([ '--marketplace-root', '--plugin', + '--expected-plugin-root', + '--expected-plugin-version', '--codex', '--codex-version', '--cwd', @@ -29,6 +31,7 @@ const VALUE_FLAGS = new Set([ ]); const BOOLEAN_FLAGS = new Set(['--help', '--quiet']); const EXACT_VERSION = /^\d+\.\d+\.\d+$/; +const EXACT_PLUGIN_VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/; const RECEIPT_STATUSES = new Set([ 'PASS', 'FAIL', @@ -50,6 +53,8 @@ const STRICT_TIMEOUT_MS = 90_000; const HELP = `Usage: codex-plugin-check --marketplace-root --plugin --codex-version [options] Options: + --expected-plugin-root Bind the local plugin source subtree + --expected-plugin-version Bind the installed plugin version --codex Codex binary for env diagnostics (default: codex) --cwd Probe workspace (default: marketplace root) --output Receipt path (default: conformance.json) @@ -86,6 +91,19 @@ function parseArguments(argv) { if (!EXACT_VERSION.test(parsed['codex-version'])) { throw new Error('--codex-version must be an exact stable numeric version'); } + const hasExpectedPluginRoot = parsed['expected-plugin-root'] !== undefined; + const hasExpectedPluginVersion = parsed['expected-plugin-version'] !== undefined; + if (hasExpectedPluginRoot !== hasExpectedPluginVersion) { + throw new Error( + '--expected-plugin-root and --expected-plugin-version must be supplied together' + ); + } + if ( + hasExpectedPluginVersion && + !EXACT_PLUGIN_VERSION.test(parsed['expected-plugin-version']) + ) { + throw new Error('--expected-plugin-version must be an exact plugin version'); + } parsed.codex ??= 'codex'; parsed.output ??= 'conformance.json'; parsed.isolation ??= 'strict'; @@ -414,6 +432,8 @@ async function runEnv(options, io, dependencies) { codexVersion: options.codexVersion, cwd: options.cwd, output: options.output, + expectedPluginRoot: options.expectedPluginRoot, + expectedPluginVersion: options.expectedPluginVersion, isolation: 'env' }); const code = validateReceipt(receipt, { @@ -450,6 +470,9 @@ async function runStrict(options, io, dependencies) { const strictPlatform = `${dependencies.platform ?? process.platform}-${ dependencies.architecture ?? process.arch }`; + const containerExpectedPluginRoot = options.expectedPluginRoot === undefined + ? undefined + : containerWorkspacePath(options.marketplaceRoot, options.expectedPluginRoot); const isolation = await createIsolation({ targetRoot: options.marketplaceRoot, receiptPath: options.output, @@ -465,6 +488,12 @@ async function runStrict(options, io, dependencies) { '/tool/src/cli.mjs', '--marketplace-root', '/workspace', '--plugin', options.plugin, + ...(containerExpectedPluginRoot === undefined + ? [] + : [ + '--expected-plugin-root', containerExpectedPluginRoot, + '--expected-plugin-version', options.expectedPluginVersion + ]), '--codex', '/usr/local/bin/codex', '--codex-version', options.codexVersion, '--cwd', containerCwd, @@ -544,9 +573,23 @@ export async function main(argv, io = process, dependencies = {}) { const cwd = await resolveRealPath( path.resolve(baseDirectory, parsed.cwd ?? marketplaceRoot) ); + const expectedPluginRoot = parsed['expected-plugin-root'] === undefined + ? undefined + : await resolveRealPath(path.resolve( + marketplaceRoot, + parsed['expected-plugin-root'] + )); + if ( + expectedPluginRoot !== undefined && + !pathIsWithin(expectedPluginRoot, marketplaceRoot) + ) { + throw new Error('--expected-plugin-root must be inside the marketplace root'); + } const options = { marketplaceRoot, plugin: parsed.plugin, + expectedPluginRoot, + expectedPluginVersion: parsed['expected-plugin-version'], codex: resolveBinary(parsed.codex, baseDirectory), codexVersion: parsed['codex-version'], cwd, diff --git a/test/check-plugin.test.mjs b/test/check-plugin.test.mjs index a540933..3efdc1f 100644 --- a/test/check-plugin.test.mjs +++ b/test/check-plugin.test.mjs @@ -182,6 +182,34 @@ test('runs the exact Codex command and app-server discovery sequence', async () ]); }); +test('binds the installed plugin source subtree and version when requested', async () => { + const expected = harness(); + const receipt = await checkPlugin({ + ...options, + expectedPluginRoot: pluginRoot, + expectedPluginVersion: '1.0.0' + }, expected.dependencies); + assert.equal(receipt.status, 'PASS'); + + for (const changed of [ + { expectedPluginVersion: '9.9.9' }, + { expectedPluginRoot: path.join(fixtureRoot, '.agents') } + ]) { + const observed = harness(); + await assert.rejects(checkPlugin({ + ...options, + expectedPluginRoot: pluginRoot, + expectedPluginVersion: '1.0.0', + ...changed + }, observed.dependencies), /expected|plugin.*(?:source|root|version)|(?:source|root|version).*plugin/i); + assert.deepEqual( + observed.lifecycle.filter(([name]) => ['assertCheckoutUnchanged', 'cleanup'].includes(name)) + .map(([name]) => name), + ['assertCheckoutUnchanged', 'cleanup'] + ); + } +}); + test('canonicalizes a symlink marketplace root across every Codex boundary', async () => { const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'codex-plugin-check-symlink-')); const linkedRoot = path.join(temporaryRoot, 'marketplace'); diff --git a/test/cli.test.mjs b/test/cli.test.mjs index 710f8fd..9a2bc08 100644 --- a/test/cli.test.mjs +++ b/test/cli.test.mjs @@ -243,7 +243,18 @@ test('required values and exact numeric Codex versions fail before dependencies ['--plugin', 'sample', '--codex-version', '0.147.0'], ['--marketplace-root', fixture.marketplaceRoot, '--codex-version', '0.147.0'], requiredArgs(fixture, { '--codex-version': 'latest' }), - requiredArgs(fixture, { '--codex-version': '0.148.0-beta.1' }) + requiredArgs(fixture, { '--codex-version': '0.148.0-beta.1' }), + [...requiredArgs(fixture), '--expected-plugin-root', 'workspace'], + [ + ...requiredArgs(fixture), + '--expected-plugin-root', 'workspace', + '--expected-plugin-version', 'latest' + ], + [ + ...requiredArgs(fixture), + '--expected-plugin-root', '..', + '--expected-plugin-version', '1.0.0' + ] ]; for (const argv of cases) { @@ -266,7 +277,9 @@ test('env mode verifies Codex first, probes the canonical checkout, and writes d const capture = captureIo(); const code = await main(requiredArgs(fixture, { - '--marketplace-root': alias + '--marketplace-root': alias, + '--expected-plugin-root': 'workspace', + '--expected-plugin-version': '1.0.0' }), capture.io, { runProcess: async (command, args, options) => { events.push('version'); @@ -290,6 +303,8 @@ test('env mode verifies Codex first, probes the canonical checkout, and writes d assert.equal(checkOptions.codex, '/fixture/codex'); assert.equal(checkOptions.cwd, await realpath(fixture.cwd)); assert.equal(checkOptions.output, path.resolve(fixture.output)); + assert.equal(checkOptions.expectedPluginRoot, await realpath(fixture.cwd)); + assert.equal(checkOptions.expectedPluginVersion, '1.0.0'); assert.equal(await readFile(fixture.output, 'utf8'), `${JSON.stringify(expected, null, 2)}\n`); assert.equal((await stat(fixture.output)).mode & 0o777, 0o600); assert.equal(capture.stdout(), 'codex-plugin-check: PASS (sample, Codex 0.147.0)\n'); @@ -535,7 +550,9 @@ test('strict mode invokes one env-mode child with container paths and certifies const code = await main(requiredArgs(fixture, { '--isolation': 'strict', - '--output': strictOutput + '--output': strictOutput, + '--expected-plugin-root': 'workspace', + '--expected-plugin-version': '1.0.0' }), capture.io, harness.dependencies); assert.equal(code, 1); @@ -553,6 +570,8 @@ test('strict mode invokes one env-mode child with container paths and certifies '/tool/src/cli.mjs', '--marketplace-root', '/workspace', '--plugin', 'sample', + '--expected-plugin-root', '/workspace/workspace', + '--expected-plugin-version', '1.0.0', '--codex', '/usr/local/bin/codex', '--codex-version', '0.147.0', '--cwd', '/workspace/workspace', diff --git a/test/public-fixtures.test.mjs b/test/public-fixtures.test.mjs index c0cc2fe..e946019 100644 --- a/test/public-fixtures.test.mjs +++ b/test/public-fixtures.test.mjs @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; +import { EventEmitter } from 'node:events'; import { access, lstat, @@ -14,6 +15,7 @@ import { } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; +import { PassThrough } from 'node:stream'; import test from 'node:test'; import { gzipSync } from 'node:zlib'; @@ -24,6 +26,7 @@ import { CODEX_VERSIONS, extractArchiveEntries, fetchPublicArchive, + inspectDockerReal, parseTarGzipArchive, preparePublicFixture, probePublicFixtureCell, @@ -35,6 +38,7 @@ import { validateFixtureDefinitions, validatePublicReceipt, validateRelativeEvidencePath, + validateStaticPluginContract, writeAtomicEvidenceJson } from '../scripts/falsify-public-fixtures.mjs'; @@ -94,6 +98,130 @@ const FIXED_DETAILS = [ ['roadrunner-admin', '.', 'roadrunner-admin', 'roadrunner', ['LICENSE'], ['skill', 'mcp']] ]; +const FIXED_PLUGIN_CONTRACTS = [ + [ + 'bitrouter', '.agents/plugins/marketplace.json', '.', '.codex-plugin/plugin.json', '0.1.0', + 'ba604cb6d8313594bebbdbc899f930195e92f6c12fa14371764f7dec2e25d4c9', + 'ba604cb6d8313594bebbdbc899f930195e92f6c12fa14371764f7dec2e25d4c9', + [ + ['mcp', 'bitrouter', 'plugin/read'], + ['skill', 'bitrouter:bitrouter', 'plugin/read + skills/list'] + ] + ], + [ + 'oh-my-cassette', '.agents/plugins/marketplace.json', '.', '.codex-plugin/plugin.json', '0.4.14', + '9dffb7f24db16606eeb44f7a23746073716069e63e4cf58a07631c02e1f57177', + '32c159545ca3626c13dfae8f1c833e456584df10c204060539475fd6c301b8e8', + [ + ['mcp', 'cassette', 'plugin/read'], + ['skill', 'oh-my-cassette:cassette-model', 'plugin/read + skills/list'], + ['skill', 'oh-my-cassette:cassette-video-edit', 'plugin/read + skills/list'] + ] + ], + [ + 'watercooler', '.agents/plugins/marketplace.json', 'plugins/codex/watercooler', + '.codex-plugin/plugin.json', '0.5.6', + 'ad91f57b0605df94a1361aa67c3b35314f5357583aaef7f364b397da5a00ea19', + 'ad91f57b0605df94a1361aa67c3b35314f5357583aaef7f364b397da5a00ea19', + [ + ['mcp', 'watercooler', 'plugin/read'], + ['skill', 'watercooler:find-related', 'plugin/read + skills/list'], + ['skill', 'watercooler:recall', 'plugin/read + skills/list'], + ['skill', 'watercooler:search-threads', 'plugin/read + skills/list'], + ['skill', 'watercooler:threads', 'plugin/read + skills/list'], + ['skill', 'watercooler:update-agent-context', 'plugin/read + skills/list'], + ['skill', 'watercooler:watercooler-health', 'plugin/read + skills/list'], + ['skill', 'watercooler:watercooler-onboarding', 'plugin/read + skills/list'] + ] + ], + [ + 'commercetools', '.agents/plugins/marketplace.json', '.agents/plugins/commercetools', + '.codex-plugin/plugin.json', '0.14.0', + '749bea75c483aecfcc71dc04919a13e170953996eeba9dfdff16c4f5f49073c0', + '749bea75c483aecfcc71dc04919a13e170953996eeba9dfdff16c4f5f49073c0', + [ + ['mcp', 'commerce-mcp', 'plugin/read'], + ['mcp', 'commercetools-knowledge', 'plugin/read'], + ['skill', 'commercetools:commercetools-checkout', 'plugin/read + skills/list'], + ['skill', 'commercetools:commercetools-commerce-patterns', 'plugin/read + skills/list'], + ['skill', 'commercetools:commercetools-connect', 'plugin/read + skills/list'], + ['skill', 'commercetools:commercetools-platform', 'plugin/read + skills/list'], + ['skill', 'commercetools:commercetools-storefront', 'plugin/read + skills/list'] + ] + ], + [ + 'ctx', '.agents/plugins/marketplace.json', '.', '.codex-plugin/plugin.json', '0.4.0', + '7f6934a57be05a126b968a5c5d346fb9d3150bdb6a57e199d2274204c94337eb', + '7b3212dbd512ee0bbf7f8c3c2b69c86bdba41ed55f9f71e5f69e63dc0cce49f7', + [ + ['hook', 'ctx@ctx-local:hooks/hooks.json:post_tool_use:0:0', 'plugin/read + hooks/list'], + ['hook', 'ctx@ctx-local:hooks/hooks.json:session_start:0:0', 'plugin/read + hooks/list'], + ['hook', 'ctx@ctx-local:hooks/hooks.json:stop:0:0', 'plugin/read + hooks/list'], + ['skill', 'ctx:ctx', 'plugin/read + skills/list'] + ] + ], + [ + 'agentmail', '.agents/plugins/marketplace.json', '.', '.codex-plugin/plugin.json', '0.3.0', + '97a82ec2aaa745663a6baa0dab16c476858d4ddc47230f2906b26eafbffa6669', + '97a82ec2aaa745663a6baa0dab16c476858d4ddc47230f2906b26eafbffa6669', + [ + ['mcp', 'agentmail', 'plugin/read'], + ['skill', 'agentmail:agent-email-patterns', 'plugin/read + skills/list'], + ['skill', 'agentmail:agentmail', 'plugin/read + skills/list'], + ['skill', 'agentmail:agentmail-cli', 'plugin/read + skills/list'], + ['skill', 'agentmail:agentmail-mcp', 'plugin/read + skills/list'], + ['skill', 'agentmail:agentmail-toolkit', 'plugin/read + skills/list'], + ['skill', 'agentmail:check-email', 'plugin/read + skills/list'], + ['skill', 'agentmail:manage-inboxes', 'plugin/read + skills/list'], + ['skill', 'agentmail:send-email', 'plugin/read + skills/list'] + ] + ], + [ + 'sarathi', '.agents/plugins/marketplace.json', '.', '.codex-plugin/plugin.json', '0.6.0', + 'da617a7857381c86ef963f85185a1afef851369a52cb4630d9360c21df904599', + 'da617a7857381c86ef963f85185a1afef851369a52cb4630d9360c21df904599', + [['skill', 'sarathi:sarathi', 'plugin/read + skills/list']] + ], + [ + 'cc-plugin-codex', '.agents/plugins/marketplace.json', 'plugins/cc-plugin-codex', + '.codex-plugin/plugin.json', '0.1.1', + '1da66cadeb01bb4b57f63e522fdc763df98d5ba6e2034e04599b02c1e394daed', + '1da66cadeb01bb4b57f63e522fdc763df98d5ba6e2034e04599b02c1e394daed', + [ + ['hook', 'cc-plugin-codex@cc-plugin-codex:hooks/hooks.json:session_end:0:0', 'plugin/read + hooks/list'], + ['hook', 'cc-plugin-codex@cc-plugin-codex:hooks/hooks.json:stop:0:0', 'plugin/read + hooks/list'], + ['skill', 'cc-plugin-codex:claude-adversarial-review', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-cancel', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-cli-runtime', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-prompting', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-rescue', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-result', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-result-handling', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-review', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-setup', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-status', 'plugin/read + skills/list'], + ['skill', 'cc-plugin-codex:claude-transfer', 'plugin/read + skills/list'] + ] + ], + [ + 'speedy-skills', '.claude-plugin/marketplace.json', 'plugins/example-minimal', + '.claude-plugin/plugin.json', '0.1.0', + 'd39797da9765bf1d822887dc6735f186d4bfc199279558817cf6eef375aab1c2', + 'd39797da9765bf1d822887dc6735f186d4bfc199279558817cf6eef375aab1c2', + [['skill', 'example-minimal:summarizing-git-log', 'plugin/read + skills/list']] + ], + [ + 'roadrunner-admin', '.agents/plugins/marketplace.json', 'plugins/roadrunner-admin', + '.codex-plugin/plugin.json', '0.1.0', + '33eaa8e5aef5449d496771317d1f40205dabf06fa6d6041e1e142e61e651f5d1', + '33eaa8e5aef5449d496771317d1f40205dabf06fa6d6041e1e142e61e651f5d1', + [ + ['mcp', 'roadrunner-admin', 'plugin/read'], + ['skill', 'roadrunner-admin:roadrunner-admin', 'plugin/read + skills/list'] + ] + ] +]; + async function temporaryDirectory(t, prefix) { const directory = await realpath(await mkdtemp(path.join(os.tmpdir(), prefix))); t.after(() => rm(directory, { recursive: true, force: true })); @@ -111,14 +239,10 @@ function publicReceipt(fixture, version, overrides = {}) { marketplace: fixture.marketplace, sourceRoot: '/workspace' }, - capabilities: fixture.expectedKinds.map((kind, index) => ({ + capabilities: fixture.expectedCapabilities.map(({ kind, key, source }) => ({ kind, - key: `${fixture.plugin}:${kind}:${index}`, - source: kind === 'skill' - ? 'plugin/read + skills/list' - : kind === 'hook' - ? 'plugin/read + hooks/list' - : 'plugin/read', + key, + source, status: kind === 'skill' ? 'DISCOVERED_EFFECTIVE' : kind === 'hook' @@ -130,6 +254,32 @@ function publicReceipt(fixture, version, overrides = {}) { }; } +async function preparedFixtureStub(fixture, temporaryRoot) { + const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); + await mkdir(marketplaceRoot); + return { + archiveSha256: 'a'.repeat(64), + checkoutSha256: fixture.expectedCheckoutSha256, + marketplaceSha256: fixture.expectedMarketplaceSha256, + marketplaceRoot, + adapter: fixture.adapterId === 'none' + ? null + : { + id: 'local-source-v1', + originalSha256: 'd'.repeat(64), + adaptedSha256: 'e'.repeat(64) + }, + pluginContract: { + marketplaceManifestPath: fixture.marketplaceManifestPath, + marketplace: fixture.marketplace, + plugin: fixture.plugin, + pluginRoot: fixture.pluginRoot, + pluginManifestPath: fixture.pluginManifestPath, + pluginVersion: fixture.pluginVersion + } + }; +} + function writeTarString(header, offset, length, value) { Buffer.from(value).copy(header, offset, 0, length); } @@ -193,6 +343,19 @@ test('registry binds exactly the ten audited commits and only one static adapter }) => [evidenceId, marketplaceRoot, plugin, marketplace, licensePaths, expectedKinds]), FIXED_DETAILS ); + assert.deepEqual( + PUBLIC_FIXTURES.map((fixture) => [ + fixture.evidenceId, + fixture.marketplaceManifestPath, + fixture.pluginRoot, + fixture.pluginManifestPath, + fixture.pluginVersion, + fixture.expectedCheckoutSha256, + fixture.expectedMarketplaceSha256, + fixture.expectedCapabilities.map(({ kind, key, source }) => [kind, key, source]) + ]), + FIXED_PLUGIN_CONTRACTS + ); for (const fixture of PUBLIC_FIXTURES) { assert.equal(fixture.repositoryUrl, `https://github.com/${fixture.repository}`); assert.equal( @@ -225,7 +388,31 @@ test('environment accepts only exact released versions and a checkout-relative o architecture: process.arch, outputBoundary: '/checkout', outputRoot: '/checkout/artifacts/public-fixtures', - platform: process.platform + platform: process.platform, + provenance: null + }); + + const github = { + ...env, + GITHUB_ACTIONS: 'true', + GITHUB_EVENT_NAME: 'push', + GITHUB_REF: 'refs/heads/main', + GITHUB_REPOSITORY: 'builtbyhuy/codex-plugin-check', + GITHUB_RUN_ATTEMPT: '2', + GITHUB_RUN_ID: '31379758041', + GITHUB_SERVER_URL: 'https://github.com', + GITHUB_SHA: 'a'.repeat(40) + }; + assert.deepEqual(publicFixtureOptionsFromEnvironment(github, '/checkout').provenance, { + provider: 'github-actions', + repository: 'builtbyhuy/codex-plugin-check', + commit: 'a'.repeat(40), + ref: 'refs/heads/main', + event: 'push', + runId: '31379758041', + runAttempt: '2', + runUrl: 'https://github.com/builtbyhuy/codex-plugin-check/actions/runs/31379758041', + artifactName: 'public-fixture-evidence-31379758041-2' }); for (const invalid of [ @@ -233,11 +420,15 @@ test('environment accepts only exact released versions and a checkout-relative o { ...env, CODEX_PRIOR_VERSION: '0.146.0' }, { ...env, CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: '/tmp/evidence' }, { ...env, CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: '../evidence' }, - { ...env, CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: 'artifacts/../../evidence' } + { ...env, CODEX_PUBLIC_FIXTURE_OUTPUT_ROOT: 'artifacts/../../evidence' }, + { ...github, GITHUB_REF: 'refs/heads/feature' }, + { ...github, GITHUB_RUN_ID: 'not-a-run' }, + { ...github, GITHUB_SHA: 'short' }, + { ...github, GITHUB_ACTIONS: undefined } ]) { assert.throws( () => publicFixtureOptionsFromEnvironment(invalid, '/checkout'), - /0\.147\.0|0\.146\.1|relative evidence directory/i + /0\.147\.0|0\.146\.1|relative evidence directory|GitHub.*provenance/i ); } }); @@ -343,6 +534,24 @@ test('receipt validator binds strict identity and audited capability semantics', /personal|privacy|host path/i ); + const many = PUBLIC_FIXTURES.find(({ evidenceId }) => evidenceId === 'watercooler'); + const complete = publicReceipt(many, '0.147.0'); + for (const capabilities of [ + complete.capabilities.filter((_, index) => index !== 2), + complete.capabilities.map((capability, index) => index === 0 + ? { ...capability, key: 'wrong-key' } + : capability), + complete.capabilities.map((capability, index) => index === 0 + ? { ...capability, source: 'wrong-source' } + : capability) + ]) { + assert.throws(() => validatePublicReceipt({ ...complete, capabilities }, { + fixture: many, + version: '0.147.0', + architecture: 'x64' + }), /capability|receipt|source|key|expected/i); + } + const failed = { ...base, status: 'FAIL', @@ -394,23 +603,7 @@ test('injected orchestration probes exactly ten by two cells and emits only sani await mkdir(scratchRoot); return scratchRoot; }, - prepareFixture: async ({ fixture, temporaryRoot }) => { - const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); - await mkdir(marketplaceRoot); - return { - archiveSha256: 'a'.repeat(64), - checkoutSha256: 'b'.repeat(64), - marketplaceSha256: 'c'.repeat(64), - marketplaceRoot, - adapter: fixture.adapterId === 'none' - ? null - : { - id: 'local-source-v1', - originalSha256: 'd'.repeat(64), - adaptedSha256: 'e'.repeat(64) - } - }; - }, + prepareFixture: ({ fixture, temporaryRoot }) => preparedFixtureStub(fixture, temporaryRoot), probeCell: async ({ fixture, version }) => { calls.push(`${fixture.repository}@${version}`); return { code: 0, receipt: publicReceipt(fixture, version) }; @@ -434,10 +627,11 @@ test('injected orchestration probes exactly ten by two cells and emits only sani observed: 10, required: 10, cells: 20, - compatibleCells: 20, - incompatibleCells: 0 + passedCells: 20, + failedCells: 0 }); assert.equal(summary.outputRoot, '.'); + assert.equal(summary.provenance, null); assert.equal(summary.fixtures.length, 10); assert.equal(summary.fixtures.flatMap(({ receipts }) => receipts).length, 20); @@ -452,6 +646,45 @@ test('injected orchestration probes exactly ten by two cells and emits only sani assert.deepEqual(JSON.parse(serialized), summary); }); +test('orchestration rejects prepared tree or plugin identity drift before probing', async (t) => { + const mutations = [ + ['checkout hash', (prepared) => { prepared.checkoutSha256 = 'f'.repeat(64); }], + ['marketplace hash', (prepared) => { prepared.marketplaceSha256 = 'f'.repeat(64); }], + ['plugin root', (prepared) => { prepared.pluginContract.pluginRoot = 'wrong-root'; }], + ['plugin version', (prepared) => { prepared.pluginContract.pluginVersion = '9.9.9'; }] + ]; + for (const [label, mutate] of mutations) { + await t.test(label, async () => { + const boundary = await temporaryDirectory(t, `public-fixture-drift-${label.replace(' ', '-')}-`); + const scratchRoot = path.join(boundary, 'scratch'); + let probeCalls = 0; + await assert.rejects(runPublicFixtureMatrix({ + architecture: 'x64', + outputBoundary: boundary, + outputRoot: path.join(boundary, 'artifacts', 'public-fixtures'), + platform: 'linux' + }, { + assertRuntime: async () => {}, + makeTemporaryRoot: async () => { + await mkdir(scratchRoot); + return scratchRoot; + }, + prepareFixture: async ({ fixture, temporaryRoot }) => { + const prepared = await preparedFixtureStub(fixture, temporaryRoot); + mutate(prepared); + return prepared; + }, + probeCell: async ({ fixture, version }) => { + probeCalls += 1; + return { code: 0, receipt: publicReceipt(fixture, version) }; + }, + removeTemporaryRoot: (root) => rm(root, { recursive: true, force: true }) + }), /prepared|hash|plugin|contract|identity/i); + assert.equal(probeCalls, 0); + }); + } +}); + test('unexpected tool exits and cleanup failures fail closed without partial evidence', async (t) => { for (const failure of ['tool', 'cleanup']) { const boundary = await temporaryDirectory(t, `public-fixture-${failure}-boundary-`); @@ -470,23 +703,7 @@ test('unexpected tool exits and cleanup failures fail closed without partial evi await mkdir(scratchRoot); return scratchRoot; }, - prepareFixture: async ({ fixture, temporaryRoot }) => { - const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); - await mkdir(marketplaceRoot); - return { - archiveSha256: 'a'.repeat(64), - checkoutSha256: 'b'.repeat(64), - marketplaceSha256: 'c'.repeat(64), - marketplaceRoot, - adapter: fixture.adapterId === 'none' - ? null - : { - id: 'local-source-v1', - originalSha256: 'd'.repeat(64), - adaptedSha256: 'e'.repeat(64) - } - }; - }, + prepareFixture: ({ fixture, temporaryRoot }) => preparedFixtureStub(fixture, temporaryRoot), probeCell: async ({ fixture, version }) => { probeCalls += 1; return failure === 'tool' @@ -503,7 +720,7 @@ test('unexpected tool exits and cleanup failures fail closed without partial evi } }); -test('a valid strict FAIL receipt is retained as version/API incompatibility and holds the summary', async (t) => { +test('a valid strict FAIL is retained without inventing a cause and holds the summary', async (t) => { const boundary = await temporaryDirectory(t, 'public-fixture-incompatible-boundary-'); const outputRoot = path.join(boundary, 'artifacts', 'public-fixtures'); const scratchParent = await temporaryDirectory(t, 'public-fixture-incompatible-scratch-'); @@ -521,23 +738,7 @@ test('a valid strict FAIL receipt is retained as version/API incompatibility and await mkdir(scratchRoot); return scratchRoot; }, - prepareFixture: async ({ fixture, temporaryRoot }) => { - const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); - await mkdir(marketplaceRoot); - return { - archiveSha256: 'a'.repeat(64), - checkoutSha256: 'b'.repeat(64), - marketplaceSha256: 'c'.repeat(64), - marketplaceRoot, - adapter: fixture.adapterId === 'none' - ? null - : { - id: 'local-source-v1', - originalSha256: 'd'.repeat(64), - adaptedSha256: 'e'.repeat(64) - } - }; - }, + prepareFixture: ({ fixture, temporaryRoot }) => preparedFixtureStub(fixture, temporaryRoot), probeCell: async ({ fixture, version }) => { const receipt = publicReceipt(fixture, version); if (!first) return { code: 0, receipt }; @@ -557,10 +758,11 @@ test('a valid strict FAIL receipt is retained as version/API incompatibility and }); assert.equal(summary.status, 'HOLD'); - assert.equal(summary.gate.incompatibleCells, 1); - assert.equal(summary.gate.compatibleCells, 19); - assert.equal(summary.fixtures[0].receipts[0].outcome, 'VERSION_OR_API_INCOMPATIBLE'); + assert.equal(summary.gate.failedCells, 1); + assert.equal(summary.gate.passedCells, 19); + assert.equal(summary.fixtures[0].receipts[0].outcome, 'FAIL'); assert.equal(summary.fixtures[0].receipts[1].outcome, 'PASS'); + assert.equal(JSON.stringify(summary).includes('VERSION_OR_API_INCOMPATIBLE'), false); assert.equal( JSON.parse(await readFile( path.join(outputRoot, summary.fixtures[0].receipts[0].receiptPath), @@ -613,17 +815,7 @@ test('swapping the created evidence root for a symlink cannot write outside', as await mkdir(scratchRoot); return scratchRoot; }, - prepareFixture: async ({ fixture, temporaryRoot }) => { - const marketplaceRoot = path.join(temporaryRoot, fixture.evidenceId); - await mkdir(marketplaceRoot); - return { - archiveSha256: 'a'.repeat(64), - checkoutSha256: 'b'.repeat(64), - marketplaceSha256: 'c'.repeat(64), - marketplaceRoot, - adapter: null - }; - }, + prepareFixture: ({ fixture, temporaryRoot }) => preparedFixtureStub(fixture, temporaryRoot), probeCell: async ({ fixture, version }) => { if (!swapped) { swapped = true; @@ -787,12 +979,31 @@ test('on-disk symlink audit rejects a lexical escape from the checkout', async ( test('real preparation verifies licenses, hashes source, and applies only the audited adapter', async (t) => { const direct = PUBLIC_FIXTURES[6]; + const directMarketplace = `${JSON.stringify({ + name: direct.marketplace, + plugins: [{ + name: direct.plugin, + source: { source: 'local', path: './' } + }] + })}\n`; + const directPluginManifest = `${JSON.stringify({ + name: direct.plugin, + version: direct.pluginVersion + })}\n`; const directArchive = tarArchive([ { path: `${direct.archiveRoot}/`, type: '5' }, { path: `${direct.archiveRoot}/LICENSE`, data: 'MIT\n' }, { path: `${direct.archiveRoot}/.agents/`, type: '5' }, { path: `${direct.archiveRoot}/.agents/plugins/`, type: '5' }, - { path: `${direct.archiveRoot}/.agents/plugins/marketplace.json`, data: '{}\n' } + { + path: `${direct.archiveRoot}/.agents/plugins/marketplace.json`, + data: directMarketplace + }, + { path: `${direct.archiveRoot}/.codex-plugin/`, type: '5' }, + { + path: `${direct.archiveRoot}/.codex-plugin/plugin.json`, + data: directPluginManifest + } ]); const directTemporaryRoot = await temporaryDirectory(t, 'public-fixture-prepare-direct-'); const directPrepared = await preparePublicFixture({ @@ -808,6 +1019,14 @@ test('real preparation verifies licenses, hashes source, and applies only the au assert.match(directPrepared.checkoutSha256, /^[a-f0-9]{64}$/); assert.match(directPrepared.marketplaceSha256, /^[a-f0-9]{64}$/); assert.equal(directPrepared.adapter, null); + assert.deepEqual(directPrepared.pluginContract, { + marketplaceManifestPath: direct.marketplaceManifestPath, + marketplace: direct.marketplace, + plugin: direct.plugin, + pluginRoot: direct.pluginRoot, + pluginManifestPath: direct.pluginManifestPath, + pluginVersion: direct.pluginVersion + }); assert.equal(await readFile(path.join(directPrepared.marketplaceRoot, 'LICENSE'), 'utf8'), 'MIT\n'); const adapted = PUBLIC_FIXTURES[1]; @@ -819,6 +1038,14 @@ test('real preparation verifies licenses, hashes source, and applies only the au { path: `${adapted.archiveRoot}/.agents/plugins/marketplace.json`, data: OH_MY_CASSETTE_MARKETPLACE + }, + { path: `${adapted.archiveRoot}/.codex-plugin/`, type: '5' }, + { + path: `${adapted.archiveRoot}/.codex-plugin/plugin.json`, + data: `${JSON.stringify({ + name: adapted.plugin, + version: adapted.pluginVersion + })}\n` } ]); const adaptedTemporaryRoot = await temporaryDirectory(t, 'public-fixture-prepare-adapted-'); @@ -843,6 +1070,33 @@ test('real preparation verifies licenses, hashes source, and applies only the au )).plugins[0].source, { source: 'local', path: './' } ); + assert.equal(adaptedPrepared.pluginContract.pluginVersion, adapted.pluginVersion); + + const directManifestPath = path.join( + directPrepared.marketplaceRoot, + direct.marketplaceManifestPath + ); + const directPluginManifestPath = path.join( + directPrepared.marketplaceRoot, + direct.pluginManifestPath + ); + await writeFile(directManifestPath, `${JSON.stringify({ + name: direct.marketplace, + plugins: [{ name: direct.plugin, source: './wrong-plugin-root' }] + })}\n`); + await assert.rejects( + validateStaticPluginContract(direct, directPrepared.marketplaceRoot), + /plugin.*root|source.*path|marketplace.*source/i + ); + await writeFile(directManifestPath, directMarketplace); + await writeFile(directPluginManifestPath, `${JSON.stringify({ + name: direct.plugin, + version: '9.9.9' + })}\n`); + await assert.rejects( + validateStaticPluginContract(direct, directPrepared.marketplaceRoot), + /plugin.*version|version.*plugin/i + ); const missingLicense = tarArchive([ { path: `${direct.archiveRoot}/`, type: '5' }, @@ -911,6 +1165,8 @@ test('production cell probe invokes the real CLI contract with a fresh staged re assert.deepEqual(calls, [[ '--marketplace-root', marketplaceRoot, '--plugin', 'bitrouter', + '--expected-plugin-root', '.', + '--expected-plugin-version', '0.1.0', '--codex-version', '0.147.0', '--cwd', marketplaceRoot, '--output', path.join(temporaryRoot, 'staged-receipts', 'bitrouter--codex-0.147.0.json'), @@ -1012,6 +1268,34 @@ test('runtime preflight hard-fails outside Linux and requires a live Docker serv } }); +test('Docker inspection escalates a timeout from SIGTERM to SIGKILL', async () => { + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + const signals = []; + child.kill = (signal) => { + signals.push(signal); + if (signal === 'SIGKILL') { + queueMicrotask(() => child.emit('close', null, 'SIGKILL')); + } + return true; + }; + + await assert.rejects(inspectDockerReal({ + killGraceMs: 5, + spawnProcess(command, args, options) { + assert.equal(command, 'docker'); + assert.deepEqual(args, ['version', '--format', '{{.Server.Version}}']); + assert.equal(options.shell, false); + return child; + }, + timeoutMs: 5 + }), /timed out after 5ms/i); + assert.deepEqual(signals, ['SIGTERM', 'SIGKILL']); + child.stdout.destroy(); + child.stderr.destroy(); +}); + test('atomic evidence creation never overwrites an existing receipt and leaves no temp file', async (t) => { const boundary = await temporaryDirectory(t, 'public-fixture-atomic-boundary-'); const outputRoot = path.join(boundary, 'evidence'); @@ -1059,7 +1343,8 @@ test('entrypoint binds exact environment options and reports failures with exit architecture: process.arch, outputBoundary: '/checkout', outputRoot: '/checkout/artifacts/public-fixtures', - platform: process.platform + platform: process.platform, + provenance: null }); const holdCode = await publicFixtureMain({ @@ -1069,7 +1354,7 @@ test('entrypoint binds exact environment options and reports failures with exit }, { runMatrix: async () => ({ status: 'HOLD' }) }); - assert.equal(holdCode, 1, 'a classified incompatibility must not make CI green'); + assert.equal(holdCode, 1, 'a failed conformance cell must not make CI green'); const toolErrorCode = await publicFixtureMain({ cwd: '/checkout', diff --git a/test/workflow-contract.test.mjs b/test/workflow-contract.test.mjs index 7ae06ce..311ef74 100644 --- a/test/workflow-contract.test.mjs +++ b/test/workflow-contract.test.mjs @@ -169,6 +169,7 @@ test('public fixture matrix runs only from trusted main pushes or manual dispatc const upload = stepUsing(source, UPLOAD); assert.equal(scalar(upload.source, 'path', 10), 'artifacts/public-fixtures'); assert.equal(scalar(upload.source, 'if-no-files-found', 10), 'error'); + assert.equal(scalar(upload.source, 'retention-days', 10), '90'); assert.ok(upload.source.includes(' if: always()')); assert.deepEqual( steps(source).filter((step) => step.source.some((line) => line.includes('uses:'))).length,