From 1bbd4a257b1878c32feba2efe7b027699986b02e Mon Sep 17 00:00:00 2001 From: Alberto Arroyo Raygada Date: Tue, 4 Aug 2026 08:27:53 -0500 Subject: [PATCH] fix(testing): the E2E board published 12 divergences that do not exist (#423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They were not product defects. All twelve had one shape — MCP FORBIDDEN while CLI and REST succeeded — and the message names the cause: Native: []. OPA: [OPA_ERROR: ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING_FLAG: A dynamic import callback was invoked without --experimental-vm-modules] Native ABAC raised NO violation. The OPA engine could not load its wasm for want of a Node flag, and dispatch requires both engines to allow, so every `tools/call` fail-closed. The capture that produced them was made on 2026-08-03 at 09:56 by a run launched WITHOUT that flag; `npm run test:exploration` does pass it. Re-run through the npm script, the same suite reports ZERO findings — measured here, not argued. So the defect is mine, in what this board did with its input. It rendered a stale capture with no indication of age, and twelve non-existent divergences read as observed fact. A derived artifact exists to make exactly that impossible. `coverage.json` carries no timestamp of its own, so the page now stamps the file's mtime and names the command that produces it. A reader can see the measurement predates the tree it describes. Four hypotheses died on the way and none was the answer: classification drift between the rego and the TS map (all twelve tools ARE classified), a role the policy does not know (`reader` needs no named role for read tools), a missing scope (none of the twelve declares one), and a harness that never authenticates (it starts the server with `allowNoAuth`). The message was in `howto-capture.json` the whole time; reading it first would have cost four fewer steps. Not changed here, and worth its own look: the exploration suite is red on this machine for an unrelated and ALREADY RECORDED reason — the how-to capture embeds rule ids that differ by environment (`DRIFT-01` on the runner, `INH-03` here), documented in GT-650. Regenerating locally fixes one half and breaks the other, so the files are left untouched. Co-authored-by: Claude Opus 5 --- .../scripts/generate-e2e-scenario-board.mjs | 34 +++++++++++++++++-- .../testing/e2e-scenario-board.es.md | 17 ++-------- .../testing/e2e-scenario-board.md | 17 ++-------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/.harness/scripts/generate-e2e-scenario-board.mjs b/.harness/scripts/generate-e2e-scenario-board.mjs index c52214bf..bca781a0 100644 --- a/.harness/scripts/generate-e2e-scenario-board.mjs +++ b/.harness/scripts/generate-e2e-scenario-board.mjs @@ -81,6 +81,29 @@ function readJson(p) { } } +/** + * When the file was written, ISO, or null. + * + * The Core's `coverage.json` carries no timestamp of its own, and the first + * version of this page therefore rendered it with no indication of age. That is + * not cosmetic: on 2026-08-03 it published 12 "cross-surface divergences" from a + * capture made hours earlier by a run whose MCP OPA engine had crashed for want + * of `--experimental-vm-modules`, so every `tools/call` fail-closed with + * FORBIDDEN. Re-run correctly, the same suite reports ZERO. The page was + * faithful to its input and the input was junk, which is exactly the failure a + * derived artifact is supposed to make impossible. + * + * Age is now stated. A reader can see at a glance that the measurement predates + * the tree it claims to describe. + */ +function mtimeIso(p) { + try { + return fs.statSync(p).mtime.toISOString(); + } catch { + return null; + } +} + function readJsonl(p) { try { return fs @@ -141,6 +164,7 @@ const T = { 'These carry a binding on every surface and no invocation reached them. They are the honest edge of this run, listed rather than rounded away.', measuredAt: 'Measured', fromRun: 'from run', + howGen: 'Produced by', }, es: { title: 'Cuadro de escenarios E2E', @@ -172,10 +196,11 @@ const T = { 'Tienen binding en cada superficie y ninguna invocación llegó a ellas. Son el borde honesto de esta corrida: se listan en vez de redondearse.', measuredAt: 'Medido', fromRun: 'de la corrida', + howGen: 'Producido por', }, }; -function render(lang, { coverage, findings, robo }) { +function render(lang, { coverage, findings, coverageAt, robo }) { const t = T[lang]; const L = []; const other = lang === 'en' ? './e2e-scenario-board.es.md' : './e2e-scenario-board.md'; @@ -200,6 +225,8 @@ function render(lang, { coverage, findings, robo }) { L.push(''); L.push(`${t.howTo}: \`npm run test:exploration\``); } else { + L.push(`${t.measuredAt} \`${coverageAt ?? '?'}\`. ${t.howGen}: \`npm run test:exploration\`.`); + L.push(''); L.push(`| ${t.metric} | ${t.value} |`); L.push('|---|---|'); L.push(`| Operations declared | ${coverage.totalOperations} |`); @@ -322,11 +349,12 @@ function main(argv = process.argv.slice(2)) { const coverage = readJson(path.join(CORE_OUT, 'coverage.json')); const findings = readJsonl(path.join(CORE_OUT, 'findings.jsonl')); + const coverageAt = mtimeIso(path.join(CORE_OUT, 'coverage.json')); const robo = latestRoboSoftRun(); const outputs = [ - [OUT_EN, render('en', { coverage, findings, robo })], - [OUT_ES, render('es', { coverage, findings, robo })], + [OUT_EN, render('en', { coverage, findings, coverageAt, robo })], + [OUT_ES, render('es', { coverage, findings, coverageAt, robo })], ]; let stale = false; diff --git a/reference/core/control-center/testing/e2e-scenario-board.es.md b/reference/core/control-center/testing/e2e-scenario-board.es.md index f3b67d66..5d44096d 100644 --- a/reference/core/control-center/testing/e2e-scenario-board.es.md +++ b/reference/core/control-center/testing/e2e-scenario-board.es.md @@ -10,6 +10,8 @@ Regenerar con: `node .harness/scripts/generate-e2e-scenario-board.mjs` ## Core — exploración cross-superficie (CLI · MCP · REST) +Medido `2026-08-04T10:24:38.541Z`. Producido por: `npm run test:exploration`. + | Medida | Valor | |---|---| | Operations declared | 73 | @@ -31,20 +33,7 @@ Tienen binding en cada superficie y ninguna invocación llegó a ellas. Son el b ### Observaciones -| Severidad | Tipo | Operación | Superficies | Observación | -|---|---|---|---|---| -| P1 | consistency | `gate-evaluate` | cli · mcp · rest | Cross-surface success divergence on gate-evaluate | -| P1 | consistency | `sdlc-status` | cli · mcp | Cross-surface success divergence on sdlc-status | -| P1 | consistency | `validate-satellite` | cli · mcp · rest | Cross-surface success divergence on validate-satellite | -| P1 | consistency | `dora-metrics` | cli · mcp | Cross-surface success divergence on dora-metrics | -| P1 | consistency | `agents-list` | cli · mcp | Cross-surface success divergence on agents-list | -| P2 | consistency | `phase-advance` | cli · mcp · rest | Cross-surface success divergence on phase-advance | -| P2 | consistency | `detect-drift` | cli · mcp · rest | Cross-surface success divergence on detect-drift | -| P2 | consistency | `evaluate` | cli · mcp · rest | Cross-surface success divergence on evaluate | -| P2 | consistency | `composable-validate` | cli · mcp · rest | Cross-surface success divergence on composable-validate | -| P2 | consistency | `recommend-topology` | cli · mcp · rest | Cross-surface success divergence on recommend-topology | -| P2 | consistency | `phase-artifacts-evaluate` | cli · mcp · rest | Cross-surface success divergence on phase-artifacts-evaluate | -| P2 | consistency | `topology-list` | mcp · rest | Cross-surface success divergence on topology-list | +Sin observaciones en la corrida registrada. ## Tracker — robots RoboSoft contra un clúster vivo diff --git a/reference/core/control-center/testing/e2e-scenario-board.md b/reference/core/control-center/testing/e2e-scenario-board.md index 32da8e31..9bbb6d44 100644 --- a/reference/core/control-center/testing/e2e-scenario-board.md +++ b/reference/core/control-center/testing/e2e-scenario-board.md @@ -10,6 +10,8 @@ Regenerate with: `node .harness/scripts/generate-e2e-scenario-board.mjs` ## Core — cross-surface exploration (CLI · MCP · REST) +Measured `2026-08-04T10:24:38.541Z`. Produced by: `npm run test:exploration`. + | Measure | Value | |---|---| | Operations declared | 73 | @@ -31,20 +33,7 @@ These carry a binding on every surface and no invocation reached them. They are ### Observations -| Severity | Type | Operation | Surfaces | Observation | -|---|---|---|---|---| -| P1 | consistency | `gate-evaluate` | cli · mcp · rest | Cross-surface success divergence on gate-evaluate | -| P1 | consistency | `sdlc-status` | cli · mcp | Cross-surface success divergence on sdlc-status | -| P1 | consistency | `validate-satellite` | cli · mcp · rest | Cross-surface success divergence on validate-satellite | -| P1 | consistency | `dora-metrics` | cli · mcp | Cross-surface success divergence on dora-metrics | -| P1 | consistency | `agents-list` | cli · mcp | Cross-surface success divergence on agents-list | -| P2 | consistency | `phase-advance` | cli · mcp · rest | Cross-surface success divergence on phase-advance | -| P2 | consistency | `detect-drift` | cli · mcp · rest | Cross-surface success divergence on detect-drift | -| P2 | consistency | `evaluate` | cli · mcp · rest | Cross-surface success divergence on evaluate | -| P2 | consistency | `composable-validate` | cli · mcp · rest | Cross-surface success divergence on composable-validate | -| P2 | consistency | `recommend-topology` | cli · mcp · rest | Cross-surface success divergence on recommend-topology | -| P2 | consistency | `phase-artifacts-evaluate` | cli · mcp · rest | Cross-surface success divergence on phase-artifacts-evaluate | -| P2 | consistency | `topology-list` | mcp · rest | Cross-surface success divergence on topology-list | +No observations in the recorded run. ## Tracker — RoboSoft robots against a live cluster