Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 101 additions & 5 deletions .harness/scripts/ci/01-validate-docs.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,90 @@ function getChangedMarkdownFilesForRendering() {
return null;
}

// GT-658: pinned on purpose. `npx -y @mermaid-js/mermaid-cli` without a version
// resolves `@latest` at the instant CI runs, so what executes is decided by
// whoever published most recently and by nothing in this repository. Bump this
// deliberately, in a commit, the way every other pinned tool here is bumped.
const MERMAID_CLI = "@mermaid-js/mermaid-cli@11.16.0";

/** The npx invocation, in one place so the preflight and the render cannot drift apart. */
function mermaidArgs(input, output) {
return ["-y", MERMAID_CLI, "-i", input, "-o", output, "-b", "transparent",
"-p", path.join(root, ".harness/scripts/puppeteer-config.json")];
}

/**
* GT-658: render one trivial diagram before rendering the corpus.
*
* On 2026-08-08 the npx install of mermaid-cli came out INCOMPLETE on a runner —
* `Cannot find package 'import-meta-resolve'`, which the package does declare, so
* the tree was half-written rather than the publish being bad. Every one of the
* 371 diagrams then failed, each reported as "mermaid render failed" against a
* document whose diagram was perfectly fine, and the job burned 23 minutes
* arriving at 371 wrong accusations.
*
* The diagram corpus and the renderer are two different things and only one of
* them can be broken by a commit. This asks which one FIRST, so a broken renderer
* costs one honest failure instead of 371 misleading ones.
*
* @returns {Promise<string|null>} an error message when the renderer cannot run
*/
function preflightMermaidRenderer(outputDirectory) {
return new Promise((resolve) => {
const input = path.join(outputDirectory, "000-preflight.mmd");
const output = path.join(outputDirectory, "000-preflight.svg");
// The smallest diagram that still exercises parse -> layout -> SVG.
fs.writeFileSync(input, "graph TD\n A[preflight] --> B[ok]\n", "utf8");

const child = spawn("npx", mermaidArgs(input, output), { encoding: "utf8" });
let stderr = "";
let stdout = "";
let settled = false;

const done = (message) => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve(message);
};

// Generous: this is the invocation that pays for the install and, on a cold
// runner, for the Chromium download too.
const timer = setTimeout(() => {
child.kill("SIGKILL");
done(`the renderer did not finish a trivial diagram within 180s (${MERMAID_CLI})`);
}, 180000);

child.stdout?.on("data", (c) => { stdout += c.toString(); });
child.stderr?.on("data", (c) => { stderr += c.toString(); });
child.on("error", (error) => done(`the renderer could not be started: ${error.message}`));
child.on("close", (status) => {
if (status !== 0) {
// mermaid-cli can exit non-zero having written NOTHING to either stream —
// observed on a machine without a usable Chromium. The first version of
// this preflight interpolated an empty string and produced "exited 1 on a
// trivial diagram: ", which is the same uninformative-message defect this
// whole guard change exists to remove. Say that it was silent, and give
// the command back so the reader can run it themselves.
const detail = (stderr || stdout).trim();
done(
`the renderer exited ${status} on a trivial diagram` +
(detail ? `: ${detail.slice(0, 600)}` : ' and printed NOTHING to stdout or stderr' +
' (a browser it cannot launch usually fails this way). Reproduce with:\n' +
` npx -y ${MERMAID_CLI} -i <file>.mmd -o out.svg -b transparent -p .harness/scripts/puppeteer-config.json`),
);
return;
}
// Exit 0 is not proof: a renderer that writes nothing has not rendered.
if (!fs.existsSync(output) || fs.statSync(output).size === 0) {
done(`the renderer exited 0 but produced no SVG for a trivial diagram (${MERMAID_CLI})`);
return;
}
done(null);
});
});
}

function renderMermaidBlock(block, outputDirectory, index) {
return new Promise((resolve) => {
const basename = `${String(index + 1).padStart(3, "0")}-${path
Expand All @@ -387,11 +471,7 @@ function renderMermaidBlock(block, outputDirectory, index) {

fs.writeFileSync(input, `${block.body}\n`, "utf8");

const child = spawn(
"npx",
["-y", "@mermaid-js/mermaid-cli", "-i", input, "-o", output, "-b", "transparent", "-p", path.join(root, ".harness/scripts/puppeteer-config.json")],
{ encoding: "utf8" },
);
const child = spawn("npx", mermaidArgs(input, output), { encoding: "utf8" });

let stdout = "";
let stderr = "";
Expand Down Expand Up @@ -450,6 +530,22 @@ async function renderMermaidBlocks() {
const blocksToRender = changedMarkdownFiles
? mermaidBlocks.filter((block) => changedMarkdownFiles.has(path.resolve(block.file)))
: mermaidBlocks;
if (blocksToRender.length === 0) {
return;
}

// GT-658: ask whether the RENDERER works before asking whether 371 diagrams do.
const rendererError = await preflightMermaidRenderer(outputDirectory);
if (rendererError) {
console.error(
`\n❌ Mermaid rendering did not run: ${rendererError}\n` +
` ${blocksToRender.length} diagram(s) were NOT checked, and none of them is implicated —\n` +
` this is the renderer, not the corpus. Nothing in this repository can fix it by\n` +
` editing a diagram; re-run, or bump the pin in ${path.relative(root, ".harness/scripts/ci/01-validate-docs.mjs")}.\n`,
);
process.exit(1);
}

const concurrency = Math.max(4, Math.min(os.cpus().length, 16));
const workers = [];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9796,6 +9796,20 @@
],
"dependencyDisposition": "accepted-scope",
"dependencyRationale": "GHSA-pm4m-ph32-ghv5 in js-yaml, reached through @nestjs/swagger, is accepted rather than fixed because no fix exists to apply: all three published swagger releases pin js-yaml exactly and all three pin a vulnerable version (11.4.4 -> 4.1.1, 11.4.5 -> 4.3.0, 11.4.6 -> 5.2.1), and npm overrides do not rewrite that nested exact spec — measured through a top-level override, a scoped override, both with the unrelated nested override objects removed, and through --package-lock-only as well as a real npm install. The acceptance is recorded in .harness/config/npm-audit-exceptions.json against the advisory AND its path, and 63-validate-npm-audit-gate fails the day it stops matching a real advisory, so it cannot outlive the hole it excuses. The three moderate advisories stay below the unchanged HIGH threshold and are untouched."
},
{
"id": "GT-658",
"closedAt": "2026-08-08",
"closureCommit": "8511e17c",
"evidence": [
".harness/scripts/ci/01-validate-docs.mjs"
],
"validationCommands": [
"node --check .harness/scripts/ci/01-validate-docs.mjs",
"node .harness/scripts/ci/01-validate-docs.mjs"
],
"dependencyDisposition": "accepted-scope",
"dependencyRationale": "The renderer stays an npx invocation rather than a lockfile dependency, pinned but not declared. Declaring @mermaid-js/mermaid-cli as a devDependency is the fuller fix -- it would put the renderer under npm audit and Dependabot and make npm ci fail loudly on the partial install that caused this -- but it drags Puppeteer and a Chromium download into every npm ci, for every developer, to render diagrams only CI renders. That trade is recorded as a separate decision rather than taken as a side effect of a bug fix. The happy path was NOT verified locally and this is stated rather than implied: mermaid-cli exits 1 with empty stdout and stderr on this machine, which has no usable Chromium; the corpus render is exercised by the Evolith Core Validation job on the runner, where it has been green on PRs 440, 441 and 442."
}
]
}
21 changes: 21 additions & 0 deletions reference/core/control-center/gaps/gap-reference-catalog.es.md
Original file line number Diff line number Diff line change
Expand Up @@ -8450,3 +8450,24 @@ La lección es la del propio tablero y esta vez la pagó quien medía: un `conta
- **El check `Trivy` reporta esto como una alerta HIGH nueva, y no es una regresión — se registra porque la reversión se intentó y hubo que deshacerla.** Trivy compara alertas sobre las líneas cambiadas, así que sustituir un id de advisory por otro en la misma posición del lockfile se lee como "1 new alert including 1 high severity security vulnerability". Medido en ambos sentidos en vez de argumentado: `/swagger.4.4` fija `js-yaml 4.1.1`, que arrastra **tres** advisories —`GHSA-52cp-r559-cp3m` (HIGH), `GHSA-5p4m-2wfm-xmqj` (HIGH) y `GHSA-h67p-54hq-rp68` (moderate)—, mientras que `11.4.6` fija `5.2.1`, que arrastra **uno**. El bump quita dos advisories high y deja uno, así que el árbol que produce es estrictamente mejor y la "alerta nueva" es el id cambiando, no un agujero abriéndose. **El casi-error es la lección:** en una primera lectura de Trivy se revirtió el bump como si él hubiera causado la alerta, y solo enumerar los advisories en AMBAS versiones mostró que la reversión empeoraba las cosas. El conteo de filas de npm dice lo contrario que el conteo de advisories: 11.4.6 produce 2 filas high (js-yaml y su padre) frente a 1 de 11.4.4, llevando un tercio de los advisories. Un campo `doNotRevertTheSwaggerBump` en la entrada de excepciones lo traslada al siguiente lector.
- **Lo que deliberadamente NO se cubre:** tres advisories `moderate` (`hono`, `@hono/node-server`, `@modelcontextprotocol/sdk`) quedan por debajo del umbral HIGH y siguen intactos, como estaban. Bajar el umbral es otra decisión y esta fila no la toma.
- **Estado:** `COMPLETADO` (2026-08-08)

#### GT-658

**Título:** Un renderizador roto acusó a cientos de diagramas inocentes, y nada del repositorio elegía qué renderizador corría

- **Propósito / Problema:** Que un guard de documentación culpe a lo que de verdad está roto, y que una herramienta sin fijar deje de decidir qué ejecuta CI.
- **Evidencia:** `01-validate-docs --render-mermaid` lanzaba `npx -y @mermaid-js/mermaid-cli` **sin versión**, una vez por diagrama. El 2026-08-08 esa instalación salió INCOMPLETA en un runner: `Cannot find package 'import-meta-resolve'` —una dependencia que `@mermaid-js/mermaid-cli@11.16.0` **sí** declara, verificado con `npm view`—, así que el fallo fue una instalación a medio escribir y no una publicación mala. Entonces falló cada diagrama como `mermaid render failed`, atribuido al documento que lo contenía. El job tardó **23m21s** en producir esos fallos, en la promoción `develop` → `main`, y el mismo comando había salido bien veinte minutos antes en una ejecución concurrente — intermitente, del entorno, e imposible de arreglar con ningún cambio en los documentos que acusaba.
- **Dos defectos en una sola invocación.**
- **El pin.** `npx -y` sin versión resuelve `@latest` cuando corre CI, así que lo que se ejecuta lo elige quien haya publicado más recientemente. Peor: la herramienta aparece **cero** veces en `package-lock.json`: no la instala `npm ci`, no la ve `npm audit`, no la sube Dependabot, y el gate de auditoría de [`GT-657`](./gap-reference-catalog.es.md#gt-657) —construido el mismo día— no puede razonar sobre ella. Un binario que descarga Chromium y corre en CI estaba fuera de todos los controles de cadena de suministro del repositorio.
- **La culpa.** El corpus de diagramas y el renderizador son cosas distintas, y solo una puede romperla un commit. El guard preguntaba "¿es válido cada uno de estos 371 diagramas?" antes de preguntar "¿funciona el renderizador?", así que un renderizador roto produjo cientos de acusaciones seguras y equivocadas, cada una nombrando un fichero y una línea cuyo contenido estaba bien.
- **Qué significa:** la señal más ruidosa apuntaba a la única parte inocente. Quien siguiera esos fallos editaría diagramas correctos buscando un defecto que nunca estuvo ahí, que es peor que no tener señal.
- **Componente:** `.harness` · **Criticidad:** P2 · **Complejidad:** S
- **Principal:** `S` · **Interés:** `MED` · **Base:** `estimate`
- **Procedencia:** Encontrado el 2026-08-08 en el CI del PR #443, la promoción que llevaba GT-622/656/657. Diagnosticado en vez de despachado como flake: la ejecución que falló era la de `push`, la del propio PR pasó, y la herramienta declara el paquete que no encontraba — tres hechos que juntos dicen "instalación incompleta", no "diagramas rotos".
- **Criterios de aceptación:**
- [x] Un renderizador que no puede correr produce UN error que lo nombra y dice explícitamente que ningún diagrama está implicado — observado fijando una versión inexistente: exit 1, **0** líneas `mermaid render failed`, y "405 diagram(s) were NOT checked, and none of them is implicated".
- [x] El exit 0 no se acepta como prueba de renderizado: un preflight que sale 0 sin escribir SVG, o con uno vacío, falla tan alto como un crash. Un renderizador que no produjo nada no ha renderizado.
- [x] La versión que ejecuta CI la elige un commit, no la publicación más reciente del registro — fijada en `11.16.0`, en una única constante que leen tanto el preflight como el render por diagrama, para que no puedan divergir.
- [x] El camino feliz sigue renderizando el corpus y el guard sigue fallando ante un diagrama de verdad malformado — la comprobación del corpus no cambia, solo el orden de las preguntas.
- **Deliberadamente NO hecho, y por qué es una decisión aparte:** declarar la herramienta como devDependency fijada es el arreglo completo — mete el renderizador en el lockfile, bajo `npm audit`, bajo Dependabot, y hace que `npm ci` falle ruidosamente ante la instalación parcial que causó esto. También arrastra Puppeteer y una descarga de Chromium a cada `npm ci`, de cada desarrollador, para renderizar diagramas que solo renderiza CI. Ese intercambio merece hacerse a propósito, no como efecto colateral de arreglar un bug.
- **Estado:** `COMPLETADO` (2026-08-08)
21 changes: 21 additions & 0 deletions reference/core/control-center/gaps/gap-reference-catalog.md
Original file line number Diff line number Diff line change
Expand Up @@ -8545,3 +8545,24 @@ The lesson is the board's own, and this time the measurer paid it: a `contains`
- **The `Trivy` check reports this as a NEW high alert, and that is not a regression — recorded because the revert was attempted and had to be undone.** Trivy compares alerts on the changed lines, so replacing one advisory id with another at the same lockfile position reads as "1 new alert including 1 high severity security vulnerability". Measured both ways rather than argued: `/swagger.4.4` pins `js-yaml 4.1.1`, which carries **three** advisories — `GHSA-52cp-r559-cp3m` (HIGH), `GHSA-5p4m-2wfm-xmqj` (HIGH) and `GHSA-h67p-54hq-rp68` (moderate) — while `11.4.6` pins `5.2.1`, which carries **one**. The bump removes two high advisories and leaves one, so the tree it produces is strictly better and the "new alert" is the id changing, not a hole opening. **The near-miss is the lesson:** on a first reading of Trivy the bump was reverted as if it had caused the alert, and only enumerating the advisories at BOTH versions showed the revert made things worse. npm's row count says the opposite of the advisory count here — 11.4.6 produces 2 high ROWS (js-yaml and its parent) against 11.4.4's 1, while carrying a third of the advisories. A `doNotRevertTheSwaggerBump` field in the exceptions entry carries this to the next reader.
- **What is deliberately NOT covered:** three `moderate` advisories (`hono`, `@hono/node-server`, `@modelcontextprotocol/sdk`) stay below the HIGH threshold and are untouched, as they were before. Raising the threshold is a separate decision and this row does not take it.
- **Status:** `DONE` (2026-08-08)

#### GT-658

**Title:** A broken renderer accused hundreds of innocent diagrams, and nothing in the repository chose which renderer ran

- **Purpose / Problem:** Make a documentation guard blame the thing that is actually broken, and stop an unpinned tool from deciding what CI executes.
- **Evidence:** `01-validate-docs --render-mermaid` spawned `npx -y @mermaid-js/mermaid-cli` with **no version**, once per diagram. On 2026-08-08 that install came out INCOMPLETE on a runner: `Cannot find package 'import-meta-resolve'` — a dependency `@mermaid-js/mermaid-cli@11.16.0` **does** declare, verified with `npm view`, so the failure was a half-written install and not a bad publish. Every diagram then failed as `mermaid render failed`, attributed to the document containing it. The job took **23m21s** to produce those failures, on the `develop` → `main` promotion, and the same command had succeeded twenty minutes earlier in a concurrent run — intermittent, environmental, and unfixable by any change to the documents it accused.
- **Two defects in one invocation.**
- **The pin.** `npx -y` with no version resolves `@latest` when CI runs, so what executes is chosen by whoever published most recently. Worse, the tool appears in `package-lock.json` **zero** times: `npm ci` does not install it, `npm audit` cannot see it, Dependabot cannot bump it, and [`GT-657`](./gap-reference-catalog.md#gt-657)'s audit gate — built the same day — cannot reason about it. A binary that downloads Chromium and runs in CI sat outside every supply-chain control this repository has.
- **The blame.** The diagram corpus and the renderer are different things, and only one of them can be broken by a commit. The guard asked "is each of these 371 diagrams valid?" before asking "does the renderer work?", so a broken renderer produced hundreds of confident, wrong accusations — each naming a file and a line whose content was fine.
- **What it means:** the loudest signal pointed at the only party that was innocent. A reader following those failures would edit correct diagrams looking for a defect that was never there, which is worse than no signal.
- **Component:** `.harness` · **Criticality:** P2 · **Complexity:** S
- **Principal:** `S` · **Interest:** `MED` · **Basis:** `estimate`
- **Provenance:** Found on 2026-08-08 in the CI of PR #443, the promotion carrying GT-622/656/657. Diagnosed rather than waved through as flake: the run that failed was the `push` build, the PR's own build passed, and the tool declares the package it could not find — three facts that together say "incomplete install", not "broken diagrams".
- **Acceptance criteria:**
- [x] A renderer that cannot run produces ONE error naming the renderer, and states explicitly that no diagram is implicated — observed by pinning a non-existent version: exit 1, **0** `mermaid render failed` lines, and "405 diagram(s) were NOT checked, and none of them is implicated".
- [x] Exit 0 is not accepted as proof of rendering: a preflight that exits 0 while writing no SVG, or an empty one, fails as loudly as a crash. A renderer that produced nothing has not rendered.
- [x] The version CI executes is chosen by a commit, not by the registry's newest publish — pinned to `11.16.0`, in one constant that both the preflight and the per-diagram render read, so the two cannot drift apart.
- [x] The happy path still renders the corpus and the guard still fails on a genuinely malformed diagram — the corpus check is unchanged, only the order of questions is.
- **Deliberately NOT done, and why it is a separate decision:** declaring the tool as a pinned devDependency is the fuller fix — it puts the renderer in the lockfile, under `npm audit`, under Dependabot, and makes `npm ci` fail loudly on the very partial install that caused this. It also drags Puppeteer and a Chromium download into every `npm ci`, for every developer, to render diagrams that only CI renders. That trade is worth making deliberately, not as a side effect of a bug fix.
- **Status:** `DONE` (2026-08-08)
Loading
Loading