From f645bc6d8adbd5aee6af6b6c8d92e243338d4650 Mon Sep 17 00:00:00 2001 From: Baptiste LAFOURCADE Date: Mon, 12 Oct 2026 11:25:04 +0200 Subject: [PATCH 1/3] fix(cli): preserve exact mutation filtering with vitest 5 --- cli/aidd_docs/memory/testing.md | 1 + cli/mutation-scopes.json | 2 +- cli/package.json | 4 +- ...tryker-mutator__vitest-runner@10.0.0.patch | 246 ++++++++++++++++++ cli/pnpm-lock.yaml | 232 ++++++----------- cli/pnpm-workspace.yaml | 3 + cli/scripts/run-mutation.d.mts | 5 + cli/scripts/run-mutation.mjs | 23 +- .../mutation-covers-source.arch.test.ts | 19 ++ cli/tests/fixtures/stryker-canary/settings.ts | 8 + .../stryker-canary/settings.unit.test.ts | 20 +- cli/tests/fixtures/stryker-filter/a.test.mjs | 3 + .../stryker-filter/a.test.mjs-extra.test.mjs | 3 + cli/tests/fixtures/stryker-filter/b.test.mjs | 3 + cli/tests/fixtures/stryker-filter/probe.mjs | 49 ++++ .../stryker-filter/register-tests.mjs | 11 + .../fixtures/stryker-filter/vitest.config.mjs | 3 + .../stryker-filter.integration.test.ts | 47 ++++ .../helpers/unloadable-file-as-failed-test.ts | 1 + ...nloadable-file-as-failed-test.unit.test.ts | 2 + cli/vitest.config.ts | 4 + cli/vitest.mutation.config.ts | 2 + 22 files changed, 527 insertions(+), 164 deletions(-) create mode 100644 cli/patches/@stryker-mutator__vitest-runner@10.0.0.patch create mode 100644 cli/tests/fixtures/stryker-filter/a.test.mjs create mode 100644 cli/tests/fixtures/stryker-filter/a.test.mjs-extra.test.mjs create mode 100644 cli/tests/fixtures/stryker-filter/b.test.mjs create mode 100644 cli/tests/fixtures/stryker-filter/probe.mjs create mode 100644 cli/tests/fixtures/stryker-filter/register-tests.mjs create mode 100644 cli/tests/fixtures/stryker-filter/vitest.config.mjs create mode 100644 cli/tests/helpers/stryker-filter.integration.test.ts diff --git a/cli/aidd_docs/memory/testing.md b/cli/aidd_docs/memory/testing.md index 298fbacb1..971ef7106 100644 --- a/cli/aidd_docs/memory/testing.md +++ b/cli/aidd_docs/memory/testing.md @@ -12,6 +12,7 @@ How this package is tested: the layers, the tools, the conventions. - `vitest`, `fast-check` for properties, `ink-testing-library` for terminal views. - `stryker` for mutation, per scope, a gate in CI on the scopes a change touches. +- Vitest 5 uses canonical `fullTestName` values for filtered runs. The pnpm patch for `@stryker-mutator/vitest-runner@10.0.0` uses that same name in runner IDs and worker coverage IDs. It is specific to Vitest 5; remove or reassess it when upgrading the runner. The canary checks nested hierarchy collisions, literal spaces and regex characters, and that each dynamic mutant runs exactly its covering test. - `knip`, `jscpd`, `biome`. ## Conventions diff --git a/cli/mutation-scopes.json b/cli/mutation-scopes.json index ccef764bf..4c6938794 100644 --- a/cli/mutation-scopes.json +++ b/cli/mutation-scopes.json @@ -1,5 +1,5 @@ { - "$comment": "The one declaration of what mutation testing covers and the floor each scope must hold. scripts/run-mutation.mjs runs a scope by name and fails below its break; tests/architecture/mutation-covers-source.arch.test.ts checks no source file falls outside both maps, that every scope declares a floor, and that package.json has a script per scope. Globs, never file lists: a list goes stale the day a file is added, and the score does not drop, because the mutants that would have died were never generated. A floor is raised to the measured score after a run, never lowered without the reason here. A scope's mutate is one glob or a list where a leading ! excludes; tools is split one scope per tool profile: a profile is a static declaration whose every mutant reruns each test that loads it, and the five together outlasted every other scope on a two-core runner; vscode's single file stays with the rest. canary mutates a fixture, not the product: its one mutant stops a module from loading, which Stryker's vitest runner scores as survived on its own; at 100 the scope fails the day that happens again, so a Stryker or vitest upgrade cannot quietly bypass tests/helpers/unloadable-file-as-failed-test.ts.", + "$comment": "The one declaration of what mutation testing covers and the floor each scope must hold. scripts/run-mutation.mjs runs a scope by name and fails below its break; tests/architecture/mutation-covers-source.arch.test.ts checks no source file falls outside both maps, that every scope declares a floor, and that package.json has a script per scope. Globs, never file lists: a list goes stale the day a file is added, and the score does not drop, because the mutants that would have died were never generated. A floor is raised to the measured score after a run, never lowered without the reason here. A scope's mutate is one glob or a list where a leading ! excludes; tools is split one scope per tool profile: a profile is a static declaration whose every mutant reruns each test that loads it, and the five together outlasted every other scope on a two-core runner; vscode's single file stays with the rest. canary mutates a fixture, not the product: its static mutant stops a module from loading, which Stryker's vitest runner scores as survived on its own; its dynamic mutants check exact selection through nested suites, repeated spaces, and regex characters. The scope requires 100 and exactly each dynamic mutant's covering test, so a Stryker or Vitest upgrade cannot quietly bypass tests/helpers/unloadable-file-as-failed-test.ts or widen or lose mutation test selection.", "scopes": { "kernel": { "mutate": "src/kernel/**/*.ts", diff --git a/cli/package.json b/cli/package.json index 79cd559dc..ecfcb9112 100644 --- a/cli/package.json +++ b/cli/package.json @@ -93,12 +93,12 @@ "@stryker-mutator/core": "^10.0.0", "@stryker-mutator/vitest-runner": "^10.0.0", "@types/node": "^26.1.1", - "@vitest/coverage-v8": "^4.1.11", + "@vitest/coverage-v8": "^5.0.0", "fast-check": "^4.7.0", "jscpd": "^5.0.0", "knip": "^6.0.0", "tsup": "^8.0.0", "typescript": "^7.0.2", - "vitest": "^4.1.11" + "vitest": "^5.0.0" } } diff --git a/cli/patches/@stryker-mutator__vitest-runner@10.0.0.patch b/cli/patches/@stryker-mutator__vitest-runner@10.0.0.patch new file mode 100644 index 000000000..d965dda66 --- /dev/null +++ b/cli/patches/@stryker-mutator__vitest-runner@10.0.0.patch @@ -0,0 +1,246 @@ +diff --git a/dist/src/stryker-setup.js b/dist/src/stryker-setup.js +index aecfacbe47b285c272c47e54b3d66a716adfdc41..91ce99f47f4435333acc4fc8681894b53dbf7abe 100644 +--- a/dist/src/stryker-setup.js ++++ b/dist/src/stryker-setup.js +@@ -56,14 +56,8 @@ else { + } + } + // Stryker disable all: this file is copied to the sandbox dir +-function collectTestName({ name, suite, }) { +- const nameParts = [name]; +- let currentSuite = suite; +- while (currentSuite) { +- nameParts.unshift(currentSuite.name); +- currentSuite = currentSuite.suite; +- } +- return nameParts.join(' ').trim(); ++function collectTestName({ fullTestName }) { ++ return fullTestName; + } + function toRawTestId(test) { + return `${test.file?.filepath ?? 'unknown.js'}#${collectTestName(test)}`; +diff --git a/dist/src/test-helpers.d.ts b/dist/src/test-helpers.d.ts +index c832dcc6dc2d5c40ca70dfe46fe0af2bbbccbb6a..13ccc605fcaedc3b7cf8837a468e759ea2d9402a 100644 +--- a/dist/src/test-helpers.d.ts ++++ b/dist/src/test-helpers.d.ts +@@ -1,7 +1,6 @@ +-import type { RunnerTestCase, RunnerTestSuite } from 'vitest'; +-export declare function collectTestName({ name, suite, }: { +- name: string; +- suite?: RunnerTestSuite; ++import type { RunnerTestCase } from 'vitest'; ++export declare function collectTestName({ fullTestName }: { ++ fullTestName: string; + }): string; + export declare function toRawTestId(test: RunnerTestCase): string; + //# sourceMappingURL=test-helpers.d.ts.map +diff --git a/dist/src/test-helpers.js b/dist/src/test-helpers.js +index 56cc786f30c524ae18ce87eef1a67fdf26b703c5..d6c03f9f32cc30eed342cee7211a4f9b9adb571e 100644 +--- a/dist/src/test-helpers.js ++++ b/dist/src/test-helpers.js +@@ -1,14 +1,8 @@ + // Don't merge this file into 'vitest-helpers.ts'! + // This file is used from the testing environment (via stryker-setup.js) and thus could be loaded into the browser (when using vitest with browser mode). + // Thus we should avoid unnecessary dependencies in this file. +-export function collectTestName({ name, suite, }) { +- const nameParts = [name]; +- let currentSuite = suite; +- while (currentSuite) { +- nameParts.unshift(currentSuite.name); +- currentSuite = currentSuite.suite; +- } +- return nameParts.join(' ').trim(); ++export function collectTestName({ fullTestName }) { ++ return fullTestName; + } + export function toRawTestId(test) { + return `${test.file?.filepath ?? 'unknown.js'}#${collectTestName(test)}`; +diff --git a/dist/src/vitest-test-runner.js b/dist/src/vitest-test-runner.js +index 84f1ca026503b8e92b7adc952ec6c6a401943656..438032a5c295cbd74e5d09577fb5cbf7a35e91d7 100644 +--- a/dist/src/vitest-test-runner.js ++++ b/dist/src/vitest-test-runner.js +@@ -91,6 +91,7 @@ export class VitestTestRunner { + ]; + project.config.browser.screenshotFailures = false; + }); ++ await this.ctx.standalone(); + if (this.log.isDebugEnabled()) { + this.log.debug(`vitest final config: ${JSON.stringify(this.ctx.config, null, 2)}`); + } +@@ -141,24 +142,33 @@ export class VitestTestRunner { + ? relatedFiles.map(normalizeFileName) + : undefined; + let testFilesToRun = explicitTestFiles; +- if (testIds.length > 0) { +- const parsedTests = testIds.map(fromTestId); +- const regexTestNameFilter = parsedTests +- .map(({ test: name }) => escapeRegExp(name)) +- .join('|'); +- const regex = new RegExp(regexTestNameFilter); +- testFilesToRun = parsedTests.map(({ file }) => file); +- this.ctx.projects.forEach((project) => { +- project.config.testNamePattern = regex; +- }); +- } +- else { +- this.ctx.projects.forEach((project) => { +- project.config.testNamePattern = undefined; +- }); ++ const namesByFile = new Map(); ++ for (const { file, test } of testIds.map(fromTestId)) { ++ const resolved = normalizeFileName(path.resolve(file)); ++ const names = namesByFile.get(resolved) ?? []; ++ names.push(escapeRegExp(test)); ++ namesByFile.set(resolved, names); + } ++ this.ctx.projects.forEach((project) => { ++ project.config.testNamePattern = undefined; ++ }); + try { +- await this.ctx.start(testFilesToRun); ++ if (namesByFile.size > 0) { ++ testFilesToRun = [...namesByFile.keys()]; ++ const specifications = await this.ctx.getRelevantTestSpecifications(testFilesToRun); ++ const selected = specifications.flatMap((specification) => { ++ const names = namesByFile.get(normalizeFileName(path.resolve(specification.moduleId))); ++ return names === undefined ? [] : [specification.project.createSpecification( ++ specification.moduleId, ++ { testNamePattern: new RegExp(`^(?:${names.join('|')})$`) }, ++ specification.pool, ++ )]; ++ }); ++ await this.ctx.runTestSpecifications(selected); ++ } ++ else { ++ await this.ctx.runTestFiles(testFilesToRun ?? []); ++ } + } + catch (error) { + if ( +diff --git a/src/stryker-setup.ts b/src/stryker-setup.ts +index ff31bed1ea6cdd23d3b9e195d53303dde43526e4..6d5445997e87e399599b2555dc7ba137bc67a792 100644 +--- a/src/stryker-setup.ts ++++ b/src/stryker-setup.ts +@@ -6,7 +6,6 @@ import { + beforeAll, + afterEach, + inject, +- RunnerTestSuite, + RunnerTestCase, + } from 'vitest'; + +@@ -73,19 +72,11 @@ if (mode === 'mutant') { + + // Stryker disable all: this file is copied to the sandbox dir + function collectTestName({ +- name, +- suite, ++ fullTestName, + }: { +- name: string; +- suite?: RunnerTestSuite; ++ fullTestName: string; + }): string { +- const nameParts = [name]; +- let currentSuite = suite; +- while (currentSuite) { +- nameParts.unshift(currentSuite.name); +- currentSuite = currentSuite.suite; +- } +- return nameParts.join(' ').trim(); ++ return fullTestName; + } + + function toRawTestId(test: RunnerTestCase): string { +diff --git a/src/test-helpers.ts b/src/test-helpers.ts +index 1cf9538c2dd926d608282b40e848833585e1ea39..03f564a2780650c8474997622e5cc3003da17909 100644 +--- a/src/test-helpers.ts ++++ b/src/test-helpers.ts +@@ -1,23 +1,15 @@ +-import type { RunnerTestCase, RunnerTestSuite } from 'vitest'; ++import type { RunnerTestCase } from 'vitest'; + + // Don't merge this file into 'vitest-helpers.ts'! + // This file is used from the testing environment (via stryker-setup.js) and thus could be loaded into the browser (when using vitest with browser mode). + // Thus we should avoid unnecessary dependencies in this file. + + export function collectTestName({ +- name, +- suite, ++ fullTestName, + }: { +- name: string; +- suite?: RunnerTestSuite; ++ fullTestName: string; + }): string { +- const nameParts = [name]; +- let currentSuite = suite; +- while (currentSuite) { +- nameParts.unshift(currentSuite.name); +- currentSuite = currentSuite.suite; +- } +- return nameParts.join(' ').trim(); ++ return fullTestName; + } + + export function toRawTestId(test: RunnerTestCase): string { +diff --git a/src/vitest-test-runner.ts b/src/vitest-test-runner.ts +index b54e16ae236118113d78931dab44a2252825cde4..f7e7a0568de3f8307ad953b3a33902bc37bc77b0 100644 +--- a/src/vitest-test-runner.ts ++++ b/src/vitest-test-runner.ts +@@ -163,6 +163,7 @@ export class VitestTestRunner implements TestRunner { + ]; + project.config.browser.screenshotFailures = false; + }); ++ await this.ctx.standalone(); + if (this.log.isDebugEnabled()) { + this.log.debug( + `vitest final config: ${JSON.stringify(this.ctx.config, null, 2)}`, +@@ -227,23 +228,32 @@ export class VitestTestRunner implements TestRunner { + ? relatedFiles.map(normalizeFileName) + : undefined; + let testFilesToRun: string[] | undefined = explicitTestFiles; +- if (testIds.length > 0) { +- const parsedTests = testIds.map(fromTestId); +- const regexTestNameFilter = parsedTests +- .map(({ test: name }) => escapeRegExp(name)) +- .join('|'); +- const regex = new RegExp(regexTestNameFilter); +- testFilesToRun = parsedTests.map(({ file }) => file); +- this.ctx!.projects.forEach((project) => { +- project.config.testNamePattern = regex; +- }); +- } else { +- this.ctx!.projects.forEach((project) => { +- project.config.testNamePattern = undefined; +- }); ++ const namesByFile = new Map(); ++ for (const { file, test } of testIds.map(fromTestId)) { ++ const resolved = normalizeFileName(path.resolve(file)); ++ const names = namesByFile.get(resolved) ?? []; ++ names.push(escapeRegExp(test)); ++ namesByFile.set(resolved, names); + } ++ this.ctx!.projects.forEach((project) => { ++ project.config.testNamePattern = undefined; ++ }); + try { +- await this.ctx!.start(testFilesToRun); ++ if (namesByFile.size > 0) { ++ testFilesToRun = [...namesByFile.keys()]; ++ const specifications = await this.ctx!.getRelevantTestSpecifications(testFilesToRun); ++ const selected = specifications.flatMap((specification) => { ++ const names = namesByFile.get(normalizeFileName(path.resolve(specification.moduleId))); ++ return names === undefined ? [] : [specification.project.createSpecification( ++ specification.moduleId, ++ { testNamePattern: new RegExp(`^(?:${names.join('|')})$`) }, ++ specification.pool, ++ )]; ++ }); ++ await this.ctx!.runTestSpecifications(selected); ++ } else { ++ await this.ctx!.runTestFiles(testFilesToRun ?? []); ++ } + } catch (error) { + if ( + // No tests found, this isn't a problem, we can continue diff --git a/cli/pnpm-lock.yaml b/cli/pnpm-lock.yaml index cef062abe..f58de6bfa 100644 --- a/cli/pnpm-lock.yaml +++ b/cli/pnpm-lock.yaml @@ -111,6 +111,9 @@ overrides: postcss: '>=8.5.10' qs: '>=6.15.2' +patchedDependencies: + '@stryker-mutator/vitest-runner@10.0.0': 9e824f46a81c33f801b2120a27ca4232020bd90800df64696bdec8980afc567f + importers: .: @@ -148,13 +151,13 @@ importers: version: 10.0.0(@types/node@26.6.4) '@stryker-mutator/vitest-runner': specifier: ^10.0.0 - version: 10.0.0(@stryker-mutator/core@10.0.0(@types/node@26.6.4))(vitest@4.1.11) + version: 10.0.0(patch_hash=9e824f46a81c33f801b2120a27ca4232020bd90800df64696bdec8980afc567f)(@stryker-mutator/core@10.0.0(@types/node@26.6.4))(vitest@5.0.3) '@types/node': specifier: ^26.1.1 version: 26.6.4 '@vitest/coverage-v8': - specifier: ^4.1.11 - version: 4.1.11(vitest@4.1.11) + specifier: ^5.0.0 + version: 5.0.3(vitest@5.0.3) fast-check: specifier: ^4.7.0 version: 4.10.2 @@ -171,8 +174,8 @@ importers: specifier: ^7.0.2 version: 7.0.2 vitest: - specifier: ^4.1.11 - version: 4.1.11(@types/node@26.6.4)(@vitest/coverage-v8@4.1.11)(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)) + specifier: ^5.0.0 + version: 5.0.3(@types/node@26.6.4)(@vitest/coverage-v8@5.0.3)(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)) packages: @@ -1335,9 +1338,6 @@ packages: resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} - '@standard-schema/spec@1.1.0': - resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} - '@stryker-mutator/api@10.0.0': resolution: {integrity: sha512-ZtAJ0ZT3MVRCWJTBE2h90XB/6E+4lifHYtcTyNG6nU2nLekPgTo4gD5esjX6Okxo1b/JB4jJzyxYB54fwKAoJw==} engines: {node: '>=22.0.0'} @@ -1502,20 +1502,25 @@ packages: cpu: [x64] os: [win32] - '@vitest/coverage-v8@4.1.11': - resolution: {integrity: sha512-8MVGEFnJIcdGjcbfKmeq8z0pZHH0JlVtoVZH9Q/qwUp6wyFnEJUBMrw9DCaj+ra3vShGmhavjalMIhPNxZAUcw==} + '@vitest/coverage-v8@5.0.3': + resolution: {integrity: sha512-+klsyz7BvT1vCU28Zkfzms1Ia78XD0V41U3FUtRaa3S+vOr/EXvK1O6BvVD7l1RzEjm6SONR2aybOvDDf9u0mQ==} peerDependencies: - '@vitest/browser': 4.1.11 - vitest: 4.1.11 + '@vitest/browser': 5.0.3 + vitest: 5.0.3 peerDependenciesMeta: '@vitest/browser': optional: true - '@vitest/expect@4.1.11': - resolution: {integrity: sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==} + '@vitest/istanbul-lib-coverage@1.0.2': + resolution: {integrity: sha512-9J/JMwOf9AoJhAywhrn7ScKTL38hsWQP/qPG60OtaAFcQ5OXPwKsxZFlbnuCKmZ61m8/lGgHYnFpdyQZUvG/iA==} + engines: {node: '>=22'} - '@vitest/mocker@4.1.11': - resolution: {integrity: sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==} + '@vitest/istanbul-lib-report@1.0.2': + resolution: {integrity: sha512-gUsfXZJbzPamoIY5TvHFiMMoXESBrUMo+xqaj+rYrWI69+EnvRlYBlP96ZnHPY4vX8kyUpgAnFUCg5wZG/HkDQ==} + engines: {node: '>=22'} + + '@vitest/mocker@5.0.3': + resolution: {integrity: sha512-T8sWAIbkSyAjkwTcaEc3Iu0o9A27X1/kdXrizhZkGuSKScRQtRzclfAMpOTcGdXCsqxeWlpGy3XjqaW8CpLORg==} peerDependencies: msw: ^2.4.9 vite: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -1525,20 +1530,8 @@ packages: vite: optional: true - '@vitest/pretty-format@4.1.11': - resolution: {integrity: sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==} - - '@vitest/runner@4.1.11': - resolution: {integrity: sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==} - - '@vitest/snapshot@4.1.11': - resolution: {integrity: sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==} - - '@vitest/spy@4.1.11': - resolution: {integrity: sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==} - - '@vitest/utils@4.1.11': - resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==} + '@vitest/spy@5.0.3': + resolution: {integrity: sha512-XhFysQTB8AZ+P4gMi+Lpo99vg2AZi0qKpaB9yXQl37+CaMEAPO3iH/wGVnSyL5MPERiLezpqTVtrR6UZH5GCXg==} acorn@8.16.0: resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} @@ -1887,9 +1880,6 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} - html-escaper@2.0.2: - resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} - human-signals@8.0.1: resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} engines: {node: '>=18.18.0'} @@ -1930,18 +1920,6 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - istanbul-lib-coverage@3.2.2: - resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} - engines: {node: '>=8'} - - istanbul-lib-report@3.0.1: - resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} - engines: {node: '>=10'} - - istanbul-reports@3.2.0: - resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} - engines: {node: '>=8'} - jiti@2.6.1: resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} hasBin: true @@ -2131,13 +2109,12 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + magic-string@1.4.3: + resolution: {integrity: sha512-z12OxmaPGE0F4xlpGdjuAUckipLpQlTAuAmyGhNoD7ODpYUzvDfvtxUbjM/jwznyP178oju/KDdyi220wNJ0RQ==} + magicast@0.5.5: resolution: {integrity: sha512-UicdXN8zQ3JHlxVq+28afMXPr1z7WNY6+7EJnzTdQWkTAlMLF5fNCCKxJHBQwGaNGR11581EiQmQzx73+MvszA==} - make-dir@4.0.0: - resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} - engines: {node: '>=10'} - math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -2347,9 +2324,6 @@ packages: resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} engines: {node: '>= 0.4'} - siginfo@2.0.0: - resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} - signal-exit@4.1.0: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} @@ -2369,9 +2343,6 @@ packages: resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} engines: {node: '>= 12'} - stackback@0.0.2: - resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - std-env@4.3.0: resolution: {integrity: sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==} @@ -2411,8 +2382,9 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} - tinybench@2.9.0: - resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + tinybench@6.2.1: + resolution: {integrity: sha512-IAbQaPJIIdhVxi0mqy9lez1wEwje0UQf9F1vwtbelEG5Nbgn0nPOxCvU+mNeNS9gyJTAhrHp74CbUaSynQpZcQ==} + engines: {node: '>=20.0.0'} tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -2549,23 +2521,23 @@ packages: yaml: optional: true - vitest@4.1.11: - resolution: {integrity: sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==} - engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + vitest@5.0.3: + resolution: {integrity: sha512-xMw97S3rjdtj5dkVat7jCsqWBpvchs3RlpQctUqwJD0KkERk40vz2fJ77lDwW/Vzh/pk18eItYAzkodhSes3jQ==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' '@opentelemetry/api': ^1.9.0 - '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 - '@vitest/browser-playwright': 4.1.11 - '@vitest/browser-preview': 4.1.11 - '@vitest/browser-webdriverio': 4.1.11 - '@vitest/coverage-istanbul': 4.1.11 - '@vitest/coverage-v8': 4.1.11 - '@vitest/ui': 4.1.11 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.3 + '@vitest/browser-preview': 5.0.3 + '@vitest/browser-webdriverio': ^5.0.0-beta.5 || >=5.0.0 + '@vitest/coverage-istanbul': 5.0.3 + '@vitest/coverage-v8': 5.0.3 + '@vitest/ui': 5.0.3 happy-dom: '*' jsdom: '*' - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: '@edge-runtime/vm': optional: true @@ -2602,9 +2574,9 @@ packages: engines: {node: '>= 8'} hasBin: true - why-is-node-running@2.3.0: - resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} - engines: {node: '>=8'} + why-is-node-running@3.2.1: + resolution: {integrity: sha512-Tb2FUhB4vUsGQlfSquQLYkApkuPAFQXGFzxWKHHumVz2dK+X1RUm/HnID4+TfIGYJ1kTcwOaCk/buYCEJr6YjQ==} + engines: {node: '>=20.11'} hasBin: true wrap-ansi@9.0.2: @@ -3550,8 +3522,6 @@ snapshots: '@sindresorhus/merge-streams@4.0.0': {} - '@standard-schema/spec@1.1.0': {} - '@stryker-mutator/api@10.0.0': dependencies: mutation-testing-metrics: 3.8.4 @@ -3609,14 +3579,14 @@ snapshots: '@stryker-mutator/util@10.0.0': {} - '@stryker-mutator/vitest-runner@10.0.0(@stryker-mutator/core@10.0.0(@types/node@26.6.4))(vitest@4.1.11)': + '@stryker-mutator/vitest-runner@10.0.0(patch_hash=9e824f46a81c33f801b2120a27ca4232020bd90800df64696bdec8980afc567f)(@stryker-mutator/core@10.0.0(@types/node@26.6.4))(vitest@5.0.3)': dependencies: '@stryker-mutator/api': 10.0.0 '@stryker-mutator/core': 10.0.0(@types/node@26.6.4) '@stryker-mutator/util': 10.0.0 semver: 7.8.5 tslib: 2.8.1 - vitest: 4.1.11(@types/node@26.6.4)(@vitest/coverage-v8@4.1.11)(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)) + vitest: 5.0.3(@types/node@26.6.4)(@vitest/coverage-v8@5.0.3)(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)) '@tybys/wasm-util@0.10.3': dependencies: @@ -3700,60 +3670,34 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true - '@vitest/coverage-v8@4.1.11(vitest@4.1.11)': + '@vitest/coverage-v8@5.0.3(vitest@5.0.3)': dependencies: '@bcoe/v8-coverage': 1.0.2 - '@vitest/utils': 4.1.11 + '@vitest/istanbul-lib-coverage': 1.0.2 + '@vitest/istanbul-lib-report': 1.0.2 ast-v8-to-istanbul: 1.0.7 - istanbul-lib-coverage: 3.2.2 - istanbul-lib-report: 3.0.1 - istanbul-reports: 3.2.0 magicast: 0.5.5 obug: 2.2.1 std-env: 4.3.0 tinyrainbow: 3.2.0 - vitest: 4.1.11(@types/node@26.6.4)(@vitest/coverage-v8@4.1.11)(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)) + vitest: 5.0.3(@types/node@26.6.4)(@vitest/coverage-v8@5.0.3)(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)) + + '@vitest/istanbul-lib-coverage@1.0.2': {} - '@vitest/expect@4.1.11': + '@vitest/istanbul-lib-report@1.0.2': dependencies: - '@standard-schema/spec': 1.1.0 - '@types/chai': 5.2.3 - '@vitest/spy': 4.1.11 - '@vitest/utils': 4.1.11 - chai: 6.3.0 - tinyrainbow: 3.2.0 + '@vitest/istanbul-lib-coverage': 1.0.2 - '@vitest/mocker@4.1.11(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0))': + '@vitest/mocker@5.0.3(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0))': dependencies: - '@vitest/spy': 4.1.11 + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.3 estree-walker: 3.0.3 - magic-string: 0.30.21 + magic-string: 1.4.3 optionalDependencies: vite: 8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0) - '@vitest/pretty-format@4.1.11': - dependencies: - tinyrainbow: 3.2.0 - - '@vitest/runner@4.1.11': - dependencies: - '@vitest/utils': 4.1.11 - pathe: 2.0.3 - - '@vitest/snapshot@4.1.11': - dependencies: - '@vitest/pretty-format': 4.1.11 - '@vitest/utils': 4.1.11 - magic-string: 0.30.21 - pathe: 2.0.3 - - '@vitest/spy@4.1.11': {} - - '@vitest/utils@4.1.11': - dependencies: - '@vitest/pretty-format': 4.1.11 - convert-source-map: 2.0.0 - tinyrainbow: 3.2.0 + '@vitest/spy@5.0.3': {} acorn@8.16.0: {} @@ -4074,7 +4018,8 @@ snapshots: gopd@1.2.0: {} - has-flag@4.0.0: {} + has-flag@4.0.0: + optional: true has-symbols@1.1.0: {} @@ -4082,8 +4027,6 @@ snapshots: dependencies: function-bind: 1.1.2 - html-escaper@2.0.2: {} - human-signals@8.0.1: {} iconv-lite@0.7.3: @@ -4111,19 +4054,6 @@ snapshots: isexe@2.0.0: {} - istanbul-lib-coverage@3.2.2: {} - - istanbul-lib-report@3.0.1: - dependencies: - istanbul-lib-coverage: 3.2.2 - make-dir: 4.0.0 - supports-color: 7.2.0 - - istanbul-reports@3.2.0: - dependencies: - html-escaper: 2.0.2 - istanbul-lib-report: 3.0.1 - jiti@2.6.1: {} jiti@2.7.0: {} @@ -4262,16 +4192,16 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.6.0 + magic-string@1.4.3: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + magicast@0.5.5: dependencies: '@babel/parser': 7.29.9 '@babel/types': 7.29.8 source-map-js: 1.2.1 - make-dir@4.0.0: - dependencies: - semver: 7.8.5 - math-intrinsics@1.1.0: {} minimalistic-assert@1.0.1: {} @@ -4533,8 +4463,6 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 - siginfo@2.0.0: {} - signal-exit@4.1.0: {} simple-git@4.0.2(supports-color@7.2.0): @@ -4553,8 +4481,6 @@ snapshots: source-map@0.7.6: {} - stackback@0.0.2: {} - std-env@4.3.0: {} string-width@7.2.0: @@ -4589,6 +4515,7 @@ snapshots: supports-color@7.2.0: dependencies: has-flag: 4.0.0 + optional: true thenify-all@1.6.0: dependencies: @@ -4598,7 +4525,7 @@ snapshots: dependencies: any-promise: 1.3.0 - tinybench@2.9.0: {} + tinybench@6.2.1: {} tinyexec@0.3.2: {} @@ -4717,31 +4644,25 @@ snapshots: jiti: 2.7.0 yaml: 2.9.0 - vitest@4.1.11(@types/node@26.6.4)(@vitest/coverage-v8@4.1.11)(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)): + vitest@5.0.3(@types/node@26.6.4)(@vitest/coverage-v8@5.0.3)(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)): dependencies: - '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)) - '@vitest/pretty-format': 4.1.11 - '@vitest/runner': 4.1.11 - '@vitest/snapshot': 4.1.11 - '@vitest/spy': 4.1.11 - '@vitest/utils': 4.1.11 + '@types/chai': 5.2.3 + '@vitest/mocker': 5.0.3(vite@8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0)) + chai: 6.3.0 es-module-lexer: 2.3.2 expect-type: 1.4.0 - magic-string: 0.30.21 + magic-string: 1.4.3 obug: 2.2.1 - pathe: 2.0.3 picomatch: 4.0.7 std-env: 4.3.0 - tinybench: 2.9.0 + tinybench: 6.2.1 tinyexec: 1.3.0 tinyglobby: 0.2.17 - tinyrainbow: 3.2.0 vite: 8.3.2(@types/node@26.6.4)(esbuild@0.27.3)(jiti@2.7.0)(yaml@2.9.0) - why-is-node-running: 2.3.0 + why-is-node-running: 3.2.1 optionalDependencies: '@types/node': 26.6.4 - '@vitest/coverage-v8': 4.1.11(vitest@4.1.11) + '@vitest/coverage-v8': 5.0.3(vitest@5.0.3) transitivePeerDependencies: - msw @@ -4753,10 +4674,7 @@ snapshots: dependencies: isexe: 2.0.0 - why-is-node-running@2.3.0: - dependencies: - siginfo: 2.0.0 - stackback: 0.0.2 + why-is-node-running@3.2.1: {} wrap-ansi@9.0.2: dependencies: diff --git a/cli/pnpm-workspace.yaml b/cli/pnpm-workspace.yaml index d59cf5c71..50bcb757f 100644 --- a/cli/pnpm-workspace.yaml +++ b/cli/pnpm-workspace.yaml @@ -22,3 +22,6 @@ overrides: allowBuilds: esbuild: true lefthook: true + +patchedDependencies: + '@stryker-mutator/vitest-runner@10.0.0': patches/@stryker-mutator__vitest-runner@10.0.0.patch diff --git a/cli/scripts/run-mutation.d.mts b/cli/scripts/run-mutation.d.mts index 3aea3128d..699f3a40f 100644 --- a/cli/scripts/run-mutation.d.mts +++ b/cli/scripts/run-mutation.d.mts @@ -10,6 +10,10 @@ export interface MutationReport { { readonly mutants: readonly { readonly status: string; + readonly id?: string; + readonly coveredBy?: readonly string[]; + readonly killedBy?: readonly string[]; + readonly testsCompleted?: number; readonly static?: boolean; readonly mutatorName?: string; readonly location?: { readonly start: { readonly line: number } }; @@ -28,6 +32,7 @@ export function strykerArgs( ): string[]; export function scoreOf(report: MutationReport): number; export function breakVerdict(score: number, declared: MutationScope): string | null; +export function canarySelectionVerdict(report: MutationReport): string | null; export function changedRanges(diff: string): string[]; export function changedArgs(ranges: readonly string[]): string[]; export function survivorsOf(report: MutationReport): string[]; diff --git a/cli/scripts/run-mutation.mjs b/cli/scripts/run-mutation.mjs index daa554615..178e2ca77 100644 --- a/cli/scripts/run-mutation.mjs +++ b/cli/scripts/run-mutation.mjs @@ -112,6 +112,25 @@ export function breakVerdict(score, declared) { return null; } +/** A passing score alone cannot prove the name filter avoided a neighbouring test. Each + * dynamic canary mutant has one covering test; its run must execute and be killed by it. */ +export function canarySelectionVerdict(report) { + for (const file of Object.values(report.files ?? {})) { + for (const mutant of file.mutants) { + if (mutant.static) continue; + if ( + mutant.coveredBy?.length !== 1 || + mutant.testsCompleted !== 1 || + mutant.killedBy?.length !== 1 || + mutant.killedBy[0] !== mutant.coveredBy[0] + ) { + return `canary mutant ${mutant.id} did not run exactly its covering test`; + } + } + } + return null; +} + /** Reads first rather than checking first: a file that vanishes in between is simply absent. */ function pruneIncrementalFile(path) { let raw; @@ -198,7 +217,9 @@ function main() { const report = JSON.parse(readFileSync(join(scopeDir, "mutation.json"), "utf8")); const score = scoreOf(report); - const verdict = breakVerdict(score, scopes[scope]); + const verdict = + breakVerdict(score, scopes[scope]) ?? + (scope === "canary" ? canarySelectionVerdict(report) : null); console.log( `\nReport: reports/mutation/${scope}/ (score ${score.toFixed(1)}, floor ${scopes[scope].break})` ); diff --git a/cli/tests/architecture/mutation-covers-source.arch.test.ts b/cli/tests/architecture/mutation-covers-source.arch.test.ts index b7487098e..e50b0868a 100644 --- a/cli/tests/architecture/mutation-covers-source.arch.test.ts +++ b/cli/tests/architecture/mutation-covers-source.arch.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import { HARNESS, scopesToRun } from "../../scripts/mutation-scopes-to-run.mjs"; import { breakVerdict, + canarySelectionVerdict, changedArgs, changedRanges, pruneIncremental, @@ -116,6 +117,24 @@ describe("mutation covers every source file", () => { }); describe("the guard itself", () => { + it("rejects a canary run that skipped its covering test or ran a neighbour", () => { + const mutant = { + id: "1", + status: "Killed", + coveredBy: ["test-1"], + killedBy: ["test-1"], + testsCompleted: 1, + }; + const report = (override = {}) => ({ + files: { canary: { mutants: [{ ...mutant, ...override }] } }, + }); + expect(canarySelectionVerdict(report())).toBeNull(); + expect(canarySelectionVerdict(report({ testsCompleted: 0 }))).toMatch(/covering test/); + expect(canarySelectionVerdict(report({ testsCompleted: 2 }))).toMatch(/covering test/); + expect(canarySelectionVerdict(report({ killedBy: ["test-2"] }))).toMatch(/covering test/); + expect(canarySelectionVerdict(report({ static: true, coveredBy: [] }))).toBeNull(); + }); + const scopes = { kernel: { mutate: "src/kernel/**/*.ts", break: 70 }, tools: { mutate: "src/contexts/tools/**/*.ts", break: 60 }, diff --git a/cli/tests/fixtures/stryker-canary/settings.ts b/cli/tests/fixtures/stryker-canary/settings.ts index 9392eeee0..0e97db42e 100644 --- a/cli/tests/fixtures/stryker-canary/settings.ts +++ b/cli/tests/fixtures/stryker-canary/settings.ts @@ -3,3 +3,11 @@ export const SETTINGS: { directory: string } = JSON.parse('{"directory":".cursor export function readDirectory(): string { return SETTINGS.directory; } + +export function readSiblingDirectory(): string { + return SETTINGS.directory; +} + +export function readLiteralDirectory(): string { + return SETTINGS.directory; +} diff --git a/cli/tests/fixtures/stryker-canary/settings.unit.test.ts b/cli/tests/fixtures/stryker-canary/settings.unit.test.ts index 85f43a698..13084be89 100644 --- a/cli/tests/fixtures/stryker-canary/settings.unit.test.ts +++ b/cli/tests/fixtures/stryker-canary/settings.unit.test.ts @@ -1,12 +1,26 @@ import { describe, expect, it } from "vitest"; -import { readDirectory, SETTINGS } from "./settings.js"; +import { readDirectory, readLiteralDirectory, readSiblingDirectory, SETTINGS } from "./settings.js"; it("reads the directory the settings declare", () => { expect(SETTINGS).toStrictEqual({ directory: ".cursor/" }); }); -describe("a mutant with a filtered test name", () => { - it("runs the test inside its suite", () => { +describe("a", () => { + describe("b", () => { + it("c", () => { + expect(readSiblingDirectory()).toBe(".cursor/"); + }); + }); +}); + +describe("a b", () => { + it("c", () => { expect(readDirectory()).toBe(".cursor/"); }); }); + +describe("a suite [regex].*", () => { + it("literal test (x)+", () => { + expect(readLiteralDirectory()).toBe(".cursor/"); + }); +}); diff --git a/cli/tests/fixtures/stryker-filter/a.test.mjs b/cli/tests/fixtures/stryker-filter/a.test.mjs new file mode 100644 index 000000000..16531e5c3 --- /dev/null +++ b/cli/tests/fixtures/stryker-filter/a.test.mjs @@ -0,0 +1,3 @@ +import { registerTests } from "./register-tests.mjs"; + +registerTests("a.test.mjs"); diff --git a/cli/tests/fixtures/stryker-filter/a.test.mjs-extra.test.mjs b/cli/tests/fixtures/stryker-filter/a.test.mjs-extra.test.mjs new file mode 100644 index 000000000..015cea921 --- /dev/null +++ b/cli/tests/fixtures/stryker-filter/a.test.mjs-extra.test.mjs @@ -0,0 +1,3 @@ +import { registerTests } from "./register-tests.mjs"; + +registerTests("a.test.mjs-extra.test.mjs"); diff --git a/cli/tests/fixtures/stryker-filter/b.test.mjs b/cli/tests/fixtures/stryker-filter/b.test.mjs new file mode 100644 index 000000000..b31024b85 --- /dev/null +++ b/cli/tests/fixtures/stryker-filter/b.test.mjs @@ -0,0 +1,3 @@ +import { registerTests } from "./register-tests.mjs"; + +registerTests("b.test.mjs"); diff --git a/cli/tests/fixtures/stryker-filter/probe.mjs b/cli/tests/fixtures/stryker-filter/probe.mjs new file mode 100644 index 000000000..6dcb77a36 --- /dev/null +++ b/cli/tests/fixtures/stryker-filter/probe.mjs @@ -0,0 +1,49 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const { VitestTestRunner } = await import(pathToFileURL(process.argv[2]).href); +const trace = join(process.cwd(), "executed.txt"); +process.env.FILTER_TRACE = trace; +writeFileSync(trace, ""); +const runner = new VitestTestRunner( + { vitest: { configFile: join(process.cwd(), "vitest.config.mjs"), related: false }, disableBail: true }, + { isDebugEnabled: () => false, warn: () => {} }, + "__stryker__" +); + +try { + await runner.init(); + const dryRun = await runner.dryRun({ files: [], coverageAnalysis: "off", timeout: 10000 }); + if (dryRun.status !== "complete") throw new Error(JSON.stringify(dryRun)); + const selected = (file, name) => { + const test = dryRun.tests.find((test) => basename(test.fileName) === file && test.name === name); + if (test === undefined) throw new Error(`Missing ${file}#${name}`); + return test.id; + }; + const runSelection = async (testFilter) => { + writeFileSync(trace, ""); + const result = await runner.mutantRun({ + activeMutant: { id: "0" }, + mutantActivation: "dynamic", + sandboxFileName: join(process.cwd(), "register-tests.mjs"), + testFilter, + hitLimit: 100, + timeout: 10000, + }); + return { + status: result.status, + nrOfTests: result.nrOfTests, + executed: readFileSync(trace, "utf8").trim().split("\n").sort(), + }; + }; + const left = selected("a.test.mjs", "left"); + const crossFile = await runSelection([left, selected("b.test.mjs", "right")]); + const exactFile = await runSelection([left]); + writeFileSync( + join(process.cwd(), "result.json"), + JSON.stringify({ dryRunCount: dryRun.tests.length, crossFile, exactFile }) + ); +} finally { + await runner.dispose(); +} diff --git a/cli/tests/fixtures/stryker-filter/register-tests.mjs b/cli/tests/fixtures/stryker-filter/register-tests.mjs new file mode 100644 index 000000000..277d0413c --- /dev/null +++ b/cli/tests/fixtures/stryker-filter/register-tests.mjs @@ -0,0 +1,11 @@ +import { appendFileSync } from "node:fs"; +import { expect, it } from "vitest"; + +export function registerTests(file) { + for (const name of ["left", "right"]) { + it(name, () => { + appendFileSync(process.env.FILTER_TRACE, `${file}#${name}\n`); + expect(true).toBe(true); + }); + } +} diff --git a/cli/tests/fixtures/stryker-filter/vitest.config.mjs b/cli/tests/fixtures/stryker-filter/vitest.config.mjs new file mode 100644 index 000000000..6a309c449 --- /dev/null +++ b/cli/tests/fixtures/stryker-filter/vitest.config.mjs @@ -0,0 +1,3 @@ +export default { + test: { include: ["*.test.mjs"], environment: "node", globals: false }, +}; diff --git a/cli/tests/helpers/stryker-filter.integration.test.ts b/cli/tests/helpers/stryker-filter.integration.test.ts new file mode 100644 index 000000000..cd8485537 --- /dev/null +++ b/cli/tests/helpers/stryker-filter.integration.test.ts @@ -0,0 +1,47 @@ +import { spawnSync } from "node:child_process"; +import { cpSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; + +const FIXTURE = fileURLToPath(new URL("../fixtures/stryker-filter/", import.meta.url)); +const CLI_ROOT = fileURLToPath(new URL("../../", import.meta.url)); +const RUNNER = join( + dirname(createRequire(import.meta.url).resolve("@stryker-mutator/vitest-runner/package.json")), + "dist/src/vitest-test-runner.js" +); + +it("Stryker applies each selected name only to its exact file", () => { + const root = mkdtempSync(join(tmpdir(), "aidd-stryker-filter-")); + try { + cpSync(FIXTURE, root, { recursive: true }); + symlinkSync(join(CLI_ROOT, "node_modules"), join(root, "node_modules"), "junction"); + const child = spawnSync(process.execPath, [join(root, "probe.mjs"), RUNNER], { + cwd: root, + env: Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.startsWith("VITEST")) + ), + encoding: "utf8", + timeout: 30000, + }); + expect(child.error, child.stderr).toBeUndefined(); + expect(child.status, child.stderr).toBe(0); + expect(JSON.parse(readFileSync(join(root, "result.json"), "utf8"))).toStrictEqual({ + dryRunCount: 6, + crossFile: { + status: "survived", + nrOfTests: 2, + executed: ["a.test.mjs#left", "b.test.mjs#right"], + }, + exactFile: { + status: "survived", + nrOfTests: 1, + executed: ["a.test.mjs#left"], + }, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/cli/tests/helpers/unloadable-file-as-failed-test.ts b/cli/tests/helpers/unloadable-file-as-failed-test.ts index e2bb2e5ee..bfb6c753a 100644 --- a/cli/tests/helpers/unloadable-file-as-failed-test.ts +++ b/cli/tests/helpers/unloadable-file-as-failed-test.ts @@ -22,6 +22,7 @@ export class UnloadableFileAsFailedTest implements Reporter { fullName: "the file loads", fullTestName: "the file loads", artifacts: [], + benchmarks: [], mode: "run", meta: {}, file, diff --git a/cli/tests/helpers/unloadable-file-as-failed-test.unit.test.ts b/cli/tests/helpers/unloadable-file-as-failed-test.unit.test.ts index e13ba8dd3..c14fd77c7 100644 --- a/cli/tests/helpers/unloadable-file-as-failed-test.unit.test.ts +++ b/cli/tests/helpers/unloadable-file-as-failed-test.unit.test.ts @@ -18,6 +18,8 @@ function testFile(state: "pass" | "fail", tasks: RunnerTestFile["tasks"] = []): type: "suite", filepath: "/repo/cli/tests/contexts/tools/domain/profiles/cursor.unit.test.ts", projectName: "unit", + concurrencyId: 0, + workerId: 0, tasks, result: { state, errors: state === "fail" ? [LOAD_ERROR] : [] }, get file() { diff --git a/cli/vitest.config.ts b/cli/vitest.config.ts index 3b886d8a5..70ae8d0e6 100644 --- a/cli/vitest.config.ts +++ b/cli/vitest.config.ts @@ -9,6 +9,7 @@ export default defineConfig({ test: { projects: [ { + extends: false, plugins: [textLoader(TEXT_EXTENSIONS)], test: { name: "unit", @@ -22,6 +23,7 @@ export default defineConfig({ }, }, { + extends: false, plugins: [textLoader(TEXT_EXTENSIONS)], test: { name: "architecture", @@ -31,6 +33,7 @@ export default defineConfig({ }, }, { + extends: false, plugins: [textLoader(TEXT_EXTENSIONS)], test: { name: "integration", @@ -45,6 +48,7 @@ export default defineConfig({ }, }, { + extends: false, plugins: [textLoader(TEXT_EXTENSIONS)], test: { name: "e2e", diff --git a/cli/vitest.mutation.config.ts b/cli/vitest.mutation.config.ts index 0d57b08b1..67d2b5f47 100644 --- a/cli/vitest.mutation.config.ts +++ b/cli/vitest.mutation.config.ts @@ -29,6 +29,7 @@ export default defineConfig({ reporters: ["default", new UnloadableFileAsFailedTest()], projects: [ { + extends: false, plugins: [textLoader(TEXT_EXTENSIONS)], test: { name: "unit", @@ -39,6 +40,7 @@ export default defineConfig({ }, }, { + extends: false, plugins: [textLoader(TEXT_EXTENSIONS)], test: { name: "integration", From 3aacc7fadf0f4baf53f15e0a68e70e32217f549d Mon Sep 17 00:00:00 2001 From: Baptiste LAFOURCADE Date: Mon, 12 Oct 2026 12:37:13 +0200 Subject: [PATCH 2/3] test(cli): assert recipe and plugin removal contracts --- ...lugin-remove-selection.integration.test.ts | 423 ++++++++++++++++++ .../recipe-validation-contract.unit.test.ts | 199 ++++++++ 2 files changed, 622 insertions(+) create mode 100644 cli/tests/contexts/framework/application/plugin/plugin-remove-selection.integration.test.ts create mode 100644 cli/tests/contexts/framework/domain/recipes/recipe-validation-contract.unit.test.ts diff --git a/cli/tests/contexts/framework/application/plugin/plugin-remove-selection.integration.test.ts b/cli/tests/contexts/framework/application/plugin/plugin-remove-selection.integration.test.ts new file mode 100644 index 000000000..0a2d24c1b --- /dev/null +++ b/cli/tests/contexts/framework/application/plugin/plugin-remove-selection.integration.test.ts @@ -0,0 +1,423 @@ +import "../../../../../src/contexts/tools/domain/profiles/claude/profile.js"; +import "../../../../../src/contexts/tools/domain/profiles/codex/profile.js"; +import "../../../../../src/contexts/tools/domain/profiles/cursor/profile.js"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { PluginRemoveUseCase } from "../../../../../src/contexts/framework/application/plugin/plugin-remove-use-case.js"; +import { Manifest } from "../../../../../src/contexts/framework/domain/manifest.js"; +import { + InstalledPlugin, + type PluginEntryData, +} from "../../../../../src/contexts/framework/domain/plugins/installed-plugin.js"; +import { PluginNotFoundError } from "../../../../../src/kernel/errors.js"; +import { CapturingLogger } from "../../../../helpers/ports/capturing-logger.js"; +import { FakeHostPluginRegistryReader } from "../../../../helpers/ports/fake-host-plugin-registry-reader.js"; +import { FakeNativePluginActivator } from "../../../../helpers/ports/fake-native-plugin-activator.js"; +import { InMemoryFileAdapter } from "../../../../helpers/ports/in-memory-file-adapter.js"; +import { InMemoryManifestRepository } from "../../../../helpers/ports/in-memory-manifest-repository.js"; + +const ROOT = "/test-project"; +const OTHER = "/another-project"; +const NAME = "wanted-plugin"; + +function plugin(overrides: Partial = {}): InstalledPlugin { + return InstalledPlugin.fromJSON({ + name: NAME, + source: { kind: "local", path: "/plugin-source" }, + version: "1.0.0", + strict: false, + files: {}, + scope: "user", + ...overrides, + }); +} + +function manifest(): Manifest { + const result = Manifest.create(); + result.addTool("cursor", "1.0.0", []); + result.addTool("claude", "1.0.0", []); + return result; +} + +describe("plugin removal selects the recorded owner, name and tool", () => { + it("detaches only the selected user plugin's project claim while preserving machine files", async () => { + const project = manifest(); + const machine = manifest(); + for (const toolId of ["cursor", "claude"] as const) { + for (const name of ["other-plugin", NAME]) { + project.addPlugin(toolId, plugin({ name })); + machine.addPlugin(toolId, plugin({ name, dependents: [ROOT, OTHER] })); + } + } + const fs = new InMemoryFileAdapter(); + const witness = join(homedir(), ".cursor", "plugins", "local", NAME, "keep.md"); + fs.setFile(witness, "machine bytes"); + const projectRepo = new InMemoryManifestRepository(project, ROOT); + const userRepo = new InMemoryManifestRepository(machine); + const remove = new PluginRemoveUseCase( + fs, + projectRepo, + new CapturingLogger(), + new Map(), + new Map(), + undefined, + undefined, + userRepo + ); + + await remove.execute({ pluginName: NAME, toolIds: ["cursor"], projectRoot: ROOT }); + + expect(project.getPlugins("cursor").map((entry) => entry.name)).toEqual(["other-plugin"]); + expect(project.getPlugins("claude").map((entry) => entry.name)).toEqual(["other-plugin", NAME]); + expect(machine.getPlugins("cursor").map((entry) => [entry.name, entry.dependents])).toEqual([ + ["other-plugin", [ROOT, OTHER]], + [NAME, [OTHER]], + ]); + expect(machine.getPlugins("claude").map((entry) => entry.dependents)).toEqual([ + [ROOT, OTHER], + [ROOT, OTHER], + ]); + expect(fs.getFile(witness)).toBe("machine bytes"); + expect(projectRepo.saveCount).toBe(1); + expect(userRepo.saveCount).toBe(1); + }); + + it("removing a project entry does not detach the same name's machine claim", async () => { + const project = manifest(); + const machine = manifest(); + project.addPlugin("cursor", plugin({ scope: "project", files: { "owned.md": "digest" } })); + project.addPlugin("cursor", plugin({ name: "other-plugin", scope: "user" })); + machine.addPlugin("cursor", plugin({ dependents: [ROOT, OTHER] })); + machine.addPlugin("cursor", plugin({ name: "other-plugin", dependents: [ROOT, OTHER] })); + const before = machine.toJSON(); + const fs = new InMemoryFileAdapter({ + [join(ROOT, "owned.md")]: "owned", + [join(ROOT, "foreign.md")]: "foreign", + }); + const userRepo = new InMemoryManifestRepository(machine); + const remove = new PluginRemoveUseCase( + fs, + new InMemoryManifestRepository(project, ROOT), + new CapturingLogger(), + new Map(), + new Map(), + undefined, + undefined, + userRepo + ); + + await remove.execute({ pluginName: NAME, toolIds: ["cursor"], projectRoot: ROOT }); + + expect(await fs.fileExists(join(ROOT, "owned.md"))).toBe(false); + expect(fs.getFile(join(ROOT, "foreign.md"))).toBe("foreign"); + expect(machine.toJSON()).toEqual(before); + expect(userRepo.saveCount).toBe(0); + expect(project.getPlugins("cursor").map((entry) => entry.name)).toEqual(["other-plugin"]); + }); + + it("user removal skips a project-scoped copy and other names with active dependents", async () => { + const project = manifest(); + project.addPlugin("cursor", plugin({ scope: "project" })); + const machine = manifest(); + machine.addPlugin("cursor", plugin({ name: "other-plugin", dependents: [OTHER] })); + machine.addPlugin("cursor", plugin()); + machine.addPlugin("claude", plugin({ scope: "project", dependents: [OTHER] })); + const projectBefore = project.toJSON(); + const otherBefore = machine.getPlugins("cursor")[0].toJSON(); + const claudeBefore = machine.getPlugins("claude")[0].toJSON(); + const userRepo = new InMemoryManifestRepository(machine); + const projectRepo = new InMemoryManifestRepository(project, ROOT); + const remove = new PluginRemoveUseCase( + new InMemoryFileAdapter(), + projectRepo, + new CapturingLogger(), + new Map(), + new Map(), + undefined, + undefined, + userRepo + ); + + await remove.execute({ pluginName: NAME, toolIds: "all", projectRoot: ROOT, scope: "user" }); + + expect(machine.getPlugins("cursor").map((entry) => entry.toJSON())).toEqual([otherBefore]); + expect(machine.getPlugins("claude").map((entry) => entry.toJSON())).toEqual([claudeBefore]); + expect(project.toJSON()).toEqual(projectBefore); + expect(projectRepo.saveCount).toBe(0); + expect(userRepo.saveCount).toBe(1); + }); + + it("user removal reports a project-only match as absent without saving either manifest", async () => { + const machine = manifest(); + machine.addPlugin("cursor", plugin({ scope: "project" })); + const before = machine.toJSON(); + const userRepo = new InMemoryManifestRepository(machine); + const projectRepo = new InMemoryManifestRepository(manifest(), ROOT); + const remove = new PluginRemoveUseCase( + new InMemoryFileAdapter(), + projectRepo, + new CapturingLogger(), + new Map(), + new Map(), + undefined, + undefined, + userRepo + ); + + await expect( + remove.execute({ pluginName: NAME, toolIds: ["cursor"], projectRoot: ROOT, scope: "user" }) + ).rejects.toThrow(PluginNotFoundError); + + expect(machine.toJSON()).toEqual(before); + expect(userRepo.saveCount).toBe(0); + expect(projectRepo.saveCount).toBe(0); + }); + + it("requires a user repository before attempting user removal", async () => { + const projectRepo = new InMemoryManifestRepository(manifest(), ROOT); + const remove = new PluginRemoveUseCase( + new InMemoryFileAdapter(), + projectRepo, + new CapturingLogger(), + new Map() + ); + + await expect( + remove.execute({ pluginName: NAME, toolIds: ["cursor"], projectRoot: ROOT, scope: "user" }) + ).rejects.toThrow("User manifest repository is required for user-scope plugin removal."); + expect(projectRepo.saveCount).toBe(0); + }); +}); + +describe("native removal preserves unconfirmed refs and addresses the recorded catalogue", () => { + const ALIAS = "local-alias"; + const HOST = "host-catalogue"; + const REF = `${NAME}@${HOST}`; + const FOREIGN = `other-plugin@${HOST}`; + const provenance = { kind: "registry" as const, source: "/plugin-source" }; + + function fixture( + options: { + available?: boolean; + fail?: boolean; + enabled?: boolean; + registrations?: boolean; + } = {} + ) { + const record = manifest(); + record.addPlugin( + "claude", + plugin({ scope: "project", marketplace: ALIAS, files: { "owned.md": "digest" } }) + ); + record.addPlugin( + "claude", + plugin({ name: "other-plugin", scope: "project", marketplace: "other-alias" }) + ); + if (options.registrations !== false) + record.setNativeRegistrations("claude", { + binary: "claude", + marketplaces: [ + { alias: "other-alias", hostName: "other-host", provenance }, + { alias: ALIAS, hostName: HOST, provenance }, + ], + pluginRefs: options.enabled === false ? [FOREIGN] : [FOREIGN, REF], + }); + const fs = new InMemoryFileAdapter({ [join(ROOT, "owned.md")]: "owned" }); + const cache = join( + homedir(), + ".claude", + "plugins", + "cache", + HOST, + NAME, + "1.0.0", + "plugin.json" + ); + const foreignCache = join( + homedir(), + ".claude", + "plugins", + "cache", + HOST, + "other-plugin", + "1.0.0", + "plugin.json" + ); + fs.setFile(cache, "target cache"); + fs.setFile(foreignCache, "foreign cache"); + const activator = new FakeNativePluginActivator({ + available: options.available ?? true, + failOnUninstall: options.fail ? [REF] : [], + }); + const logger = new CapturingLogger(); + const repo = new InMemoryManifestRepository(record, ROOT); + const remove = new PluginRemoveUseCase( + fs, + repo, + logger, + new Map([["claude", activator]]), + new Map([ + [ + "claude", + new FakeHostPluginRegistryReader({ + location: "/host/registry", + refs: new Map([[REF, { enabled: true, scope: "project" }]]), + }), + ], + ]), + undefined, + undefined, + undefined, + new Map([ + [ + "claude", + { + read: async () => ({ + location: "/host/catalogues", + entries: new Map([[HOST, provenance]]), + }), + }, + ], + ]) + ); + return { record, fs, cache, foreignCache, activator, logger, repo, remove }; + } + + it("removes the exact confirmed ref and cache while preserving sibling registrations", async () => { + const f = fixture(); + const marketplaces = f.record.getNativeRegistrations("claude")?.marketplaces; + + await f.remove.execute({ pluginName: NAME, toolIds: ["claude"], projectRoot: ROOT }); + + expect(f.activator.uninstalledPlugins).toEqual([REF]); + expect(f.activator.uninstalledPluginScopes).toEqual(["project"]); + expect(f.record.getNativeRegistrations("claude")?.pluginRefs).toEqual([FOREIGN]); + expect(f.record.getNativeRegistrations("claude")?.marketplaces).toEqual(marketplaces); + expect(f.record.getPlugins("claude").map((entry) => entry.name)).toEqual(["other-plugin"]); + expect(await f.fs.fileExists(f.cache)).toBe(false); + expect(f.fs.getFile(f.foreignCache)).toBe("foreign cache"); + expect(await f.fs.fileExists(join(ROOT, "owned.md"))).toBe(false); + expect(f.logger.infoMessages).toEqual([ + `claude: cache for '${NAME}' purged: ${join(homedir(), ".claude", "plugins", "cache", HOST, NAME)}`, + ]); + expect(f.logger.warnMessages).toEqual([]); + expect(f.repo.saveCount).toBe(1); + }); + + it.each([{ available: false }, { fail: true }])( + "retains host refs and cache when native removal is unconfirmed: %j", + async (options) => { + const f = fixture(options); + + await f.remove.execute({ pluginName: NAME, toolIds: ["claude"], projectRoot: ROOT }); + + expect(f.record.getNativeRegistrations("claude")?.pluginRefs).toEqual([FOREIGN, REF]); + expect(f.fs.getFile(f.cache)).toBe("target cache"); + expect(f.fs.getFile(f.foreignCache)).toBe("foreign cache"); + expect(f.activator.uninstalledPlugins).toEqual([]); + expect(f.logger.warnMessages).toContain( + `claude: cache for '${NAME}' left in place, its own removal was not confirmed: ${join(homedir(), ".claude", "plugins", "cache", HOST, NAME)}` + ); + expect(f.logger.infoMessages).toEqual([]); + expect(f.record.getPlugins("claude").map((entry) => entry.name)).toEqual(["other-plugin"]); + } + ); + + it("leaves an unclaimed enabled plugin on the host while removing only local files", async () => { + const f = fixture({ enabled: false }); + + await f.remove.execute({ pluginName: NAME, toolIds: ["claude"], projectRoot: ROOT }); + + expect(f.activator.uninstalledPlugins).toEqual([]); + expect(f.record.getNativeRegistrations("claude")?.pluginRefs).toEqual([FOREIGN]); + expect(f.fs.getFile(f.cache)).toBe("target cache"); + expect(f.logger.warnMessages).toEqual([ + `claude: '${REF}' is not a ref this project enabled — left enabled.`, + ]); + expect(f.logger.infoMessages).toEqual([]); + expect(await f.fs.fileExists(join(ROOT, "owned.md"))).toBe(false); + }); + + it("names the local alias when an unavailable host has no recorded catalogue", async () => { + const f = fixture({ available: false, registrations: false }); + + await f.remove.execute({ pluginName: NAME, toolIds: ["claude"], projectRoot: ROOT }); + + expect(f.activator.uninstalledPlugins).toEqual([]); + expect(f.record.getNativeRegistrations("claude")).toBeUndefined(); + expect(f.fs.getFile(f.cache)).toBe("target cache"); + expect(f.logger.warnMessages).toEqual([ + `claude CLI not found on PATH — '${NAME}@${ALIAS}' was not uninstalled from the host; its ref may remain enabled after local removal.`, + ]); + expect(f.repo.saveCount).toBe(1); + }); + + it("detaches only this project's exact machine-owned native claim without disabling the shared ref", async () => { + const project = Manifest.create(); + project.addTool("codex", "1.0.0", []); + project.addTool("cursor", "1.0.0", []); + project.addPlugin("codex", plugin({ scope: "project", marketplace: ALIAS })); + const provenance = { + kind: "effective-list" as const, + root: "/host/root", + sourceType: "local", + source: "/plugin-source", + }; + const registrations = { + binary: "codex", + marketplaces: [{ alias: ALIAS, hostName: HOST, provenance }], + pluginRefs: [FOREIGN, REF], + pluginClaims: [ + { ref: FOREIGN, dependents: [ROOT, OTHER] }, + { ref: REF, dependents: [ROOT, OTHER] }, + ], + }; + project.setNativeRegistrations("codex", registrations); + const machine = Manifest.create(); + machine.addTool("codex", "1.0.0", []); + machine.setNativeRegistrations("codex", registrations); + const userRepo = new InMemoryManifestRepository(machine); + const projectRepo = new InMemoryManifestRepository(project, ROOT); + const activator = new FakeNativePluginActivator({ available: true }); + const logger = new CapturingLogger(); + const remove = new PluginRemoveUseCase( + new InMemoryFileAdapter(), + projectRepo, + logger, + new Map([["codex", activator]]), + new Map(), + undefined, + undefined, + userRepo, + new Map([ + [ + "codex", + { + read: async () => ({ + location: "/host/catalogues", + entries: new Map([[HOST, provenance]]), + }), + }, + ], + ]) + ); + + await remove.execute({ pluginName: NAME, toolIds: "all", projectRoot: ROOT }); + + expect(activator.uninstalledPlugins).toEqual([]); + expect(machine.getNativeRegistrations("codex")?.pluginRefs).toEqual([FOREIGN, REF]); + expect(machine.getNativeRegistrations("codex")?.pluginClaims).toEqual([ + { ref: FOREIGN, dependents: [ROOT, OTHER] }, + { ref: REF, dependents: [OTHER] }, + ]); + expect(project.getNativeRegistrations("codex")?.pluginRefs).toEqual([FOREIGN]); + expect(project.getPlugins("codex")).toEqual([]); + expect(project.getPlugins("cursor")).toEqual([]); + expect(userRepo.saveCount).toBe(1); + expect(projectRepo.saveCount).toBe(1); + expect(logger.warnMessages).toEqual([ + `codex: '${REF}' is an AIDD-owned machine plugin ref — left enabled; this project's claim detaches after local removal.`, + ]); + }); +}); diff --git a/cli/tests/contexts/framework/domain/recipes/recipe-validation-contract.unit.test.ts b/cli/tests/contexts/framework/domain/recipes/recipe-validation-contract.unit.test.ts new file mode 100644 index 000000000..e9abf9aca --- /dev/null +++ b/cli/tests/contexts/framework/domain/recipes/recipe-validation-contract.unit.test.ts @@ -0,0 +1,199 @@ +import assert from "node:assert/strict"; +import { test } from "vitest"; +import { inspectRecipe } from "../../../../../src/contexts/framework/domain/recipes/recipe-validation.js"; + +const file = "publish.md"; +const example = "```bash\necho ready\n```"; + +function document(step: string, introduction = "Publish an observable result.") { + return `# Publish\n\n${introduction}\n\n## Steps to publish\n\n${step}`; +} + +function findings(content: string, rule: string) { + return inspectRecipe(file, content).findings.filter((item) => item.rule === rule); +} + +test("skipped headings name the next available level at the offending line", () => { + assert.deepEqual(findings(document(`#### 1) ✅ Publish\n${example}`), "heading-depth"), [ + { + file, + line: 7, + rule: "heading-depth", + fix: "Use level 3 or shallower after the previous heading.", + }, + ]); +}); + +test.each(["```", "~~~~", "````text", "prefix ````", "```` text"])( + "a non-closing %s fence leaves HTML comments visible as published code", + (closing) => { + const content = document(`### 1) ✅ Publish\n\`\`\`\`text\n${closing}\n`); + assert.deepEqual(findings(content, "todo"), [ + { + file, + line: 10, + rule: "todo", + fix: "Resolve this unfinished marker before publishing the recipe.", + }, + ]); + assert.equal(findings(content, "fence-balance").length, 1); + } +); + +test("a closing fence with trailing whitespace allows subsequent comments to be hidden", () => { + assert.deepEqual( + inspectRecipe(file, document("### 1) ✅ Publish\n```text\nready\n``` \n")) + .findings, + [] + ); +}); + +test("inline and multiline comments preserve visible markers and local link lines", () => { + const content = document( + `### 1) ✅ Publish\n${example}\n[Before](before.md) [After](after.md)\n {{command}}` + ); + const inspection = inspectRecipe(file, content); + assert.deepEqual(inspection.links, [ + { line: 11, target: "before.md", image: false }, + { line: 11, target: "after.md", image: false }, + ]); + assert.deepEqual(inspection.findings, [ + { + file, + line: 14, + rule: "placeholder", + fix: "Replace or remove this template placeholder.", + }, + ]); +}); + +test("whitespace before a fence language does not bypass JSON validation", () => { + const result = inspectRecipe( + file, + document("### 1) ✅ Publish\n``` JSON metadata\n{broken}\n```") + ).findings; + assert.equal(result.length, 1); + assert.equal(result[0].rule, "json-syntax"); + assert.equal(result[0].line, 8); + assert.match(result[0].fix, /^Fix the JSON example: .+/u); +}); + +test("short inline code and tildes do not open fences that expose draft comments", () => { + for (const line of ["`literal`", "~literal~"]) { + assert.deepEqual( + inspectRecipe(file, document(`### 1) ✅ Publish\n${example}\n${line}\n`)) + .findings, + [] + ); + } +}); + +test("steps are numbered only inside their outcome section", () => { + const content = `# Publish\n\nPublish an observable result.\n\n## Why\n\n### 8) 📝 Earlier context\n\nExplain when to publish.\n\n## Steps to publish\n\n### 1) ✅ Publish\n${example}\n\n## Verify\n\n### 9) 🔬 Later reference\n\nExplain how to inspect the result.`; + assert.deepEqual(inspectRecipe(file, content).findings, []); +}); + +test("headings mentioning Steps or Verify do not declare those sections", () => { + const content = document(`### 1) ✅ Publish\n${example}`).replace( + "## Steps to publish", + "## Before Verify\n\nContext.\n\n## Steps to publish" + ); + assert.deepEqual(inspectRecipe(file, `${content}\n\n## Further Steps to consider`).findings, []); +}); + +test("step headings allow spacing between the number, emoji and title", () => { + assert.deepEqual(inspectRecipe(file, document(`### 1) ✅ Publish\n${example}`)).findings, []); +}); + +test("number-like category names and malformed multi-digit steps stay distinct", () => { + const content = document(`### Release 12 notes\n\n#### 1) ✅ Publish\n${example}`); + assert.deepEqual(inspectRecipe(file, content).findings, []); + for (const heading of ["### 12 Publish", "### 12. Publish"]) { + assert.deepEqual(findings(document(heading), "step-heading"), [ + { + file, + line: 7, + rule: "step-heading", + fix: "Use `### N) Title`, or `#### N) Title` below a category.", + }, + ]); + } +}); + +test("a prior step's closing fence cannot supply the next step's example", () => { + assert.deepEqual( + findings( + document(`### 1) ✅ Publish\n${example}\n### 2) 🔬 Inspect\nExplain the result.`), + "step-example" + ), + [ + { + file, + line: 11, + rule: "step-example", + fix: "Add a fenced command/config/output, a concrete table, or an operational image.", + }, + ] + ); +}); + +test("table examples permit indentation, alignment and omitted separator-edge pipes", () => { + for (const table of [ + " | State | Meaning | \n | :--- | ---: |\n | ready | Proceed |", + "| State | Meaning |\n--- | ---\nready | Proceed", + ]) { + assert.deepEqual(inspectRecipe(file, document(`### 1) ✅ Publish\n${table}`)).findings, []); + } +}); + +test.each([ + "Prose | State | Meaning |\n| --- | --- |", + "| State | Meaning | trailing prose\n| --- | --- |", + "| State | Meaning |\nProse | --- | --- |", + "| State | Meaning |\n| - | - |", +])("a malformed table cannot stand in for a concrete example: %s", (table) => { + assert.deepEqual(findings(document(`### 1) ✅ Publish\n${table}`), "step-example"), [ + { + file, + line: 7, + rule: "step-example", + fix: "Add a fenced command/config/output, a concrete table, or an operational image.", + }, + ]); +}); + +test("description validation ignores whitespace-only lines and reports indented lists", () => { + const content = document(`### 1) ✅ Publish\n${example}`, " \n 12. Publish the result."); + assert.deepEqual(findings(content, "description"), [ + { + file, + line: 4, + rule: "description", + fix: "Put a non-empty plain description paragraph directly after the H1.", + }, + ]); +}); + +test("a scheme inside a local link fragment does not make the target external", () => { + assert.deepEqual( + inspectRecipe( + file, + document(`### 1) ✅ Publish\n${example}\n[Notes](reference.md#https:section "Details")`) + ).links, + [{ line: 11, target: "reference.md#https:section", image: false }] + ); +}); + +test("URL autolinks stay concrete while angle placeholders containing URL text are rejected", () => { + const content = document( + `### 1) ✅ Publish\n${example}\n ` + ); + assert.deepEqual(findings(content, "placeholder"), [ + { + file, + line: 11, + rule: "placeholder", + fix: "Replace or remove this angle-bracket placeholder.", + }, + ]); +}); From 81c5c4ab9a692ea258bed450106a2e839b0200e9 Mon Sep 17 00:00:00 2001 From: Baptiste LAFOURCADE Date: Mon, 12 Oct 2026 12:46:25 +0200 Subject: [PATCH 3/3] test(cli): preserve tracked files and unconfirmed native refs --- ...lugin-remove-selection.integration.test.ts | 144 +++++++++++++++++- 1 file changed, 143 insertions(+), 1 deletion(-) diff --git a/cli/tests/contexts/framework/application/plugin/plugin-remove-selection.integration.test.ts b/cli/tests/contexts/framework/application/plugin/plugin-remove-selection.integration.test.ts index 0a2d24c1b..ec2bb8f6a 100644 --- a/cli/tests/contexts/framework/application/plugin/plugin-remove-selection.integration.test.ts +++ b/cli/tests/contexts/framework/application/plugin/plugin-remove-selection.integration.test.ts @@ -1,6 +1,7 @@ import "../../../../../src/contexts/tools/domain/profiles/claude/profile.js"; import "../../../../../src/contexts/tools/domain/profiles/codex/profile.js"; import "../../../../../src/contexts/tools/domain/profiles/cursor/profile.js"; +import { createHash } from "node:crypto"; import { homedir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -46,7 +47,7 @@ describe("plugin removal selects the recorded owner, name and tool", () => { const machine = manifest(); for (const toolId of ["cursor", "claude"] as const) { for (const name of ["other-plugin", NAME]) { - project.addPlugin(toolId, plugin({ name })); + project.addPlugin(toolId, plugin({ name, files: { [`${name}/keep.md`]: "digest" } })); machine.addPlugin(toolId, plugin({ name, dependents: [ROOT, OTHER] })); } } @@ -188,6 +189,147 @@ describe("plugin removal selects the recorded owner, name and tool", () => { ).rejects.toThrow("User manifest repository is required for user-scope plugin removal."); expect(projectRepo.saveCount).toBe(0); }); + + it("explicit user removal deletes the proven machine file without touching the project copy", async () => { + const relative = `${NAME}/commands/owned.md`; + const content = "installed machine bytes"; + const machine = manifest(); + machine.addPlugin( + "cursor", + plugin({ files: { [relative]: createHash("md5").update(content).digest("hex") } }) + ); + const fs = new InMemoryFileAdapter(); + const userPath = join(homedir(), ".cursor", "plugins", "local", relative); + const projectPath = join(ROOT, relative); + fs.setFile(userPath, content); + fs.setFile(projectPath, "project bytes"); + const userRepo = new InMemoryManifestRepository(machine); + const remove = new PluginRemoveUseCase( + fs, + new InMemoryManifestRepository(manifest(), ROOT), + new CapturingLogger(), + new Map(), + new Map(), + undefined, + undefined, + userRepo + ); + + await remove.execute({ + pluginName: NAME, + toolIds: ["cursor"], + projectRoot: ROOT, + scope: "user", + }); + + expect(await fs.fileExists(userPath)).toBe(false); + expect(fs.getFile(projectPath)).toBe("project bytes"); + expect(machine.getPlugins("cursor")).toEqual([]); + expect(userRepo.saveCount).toBe(1); + }); + + it("refuses a partial user removal before deleting safe files when another tracked path escapes", async () => { + const content = "installed bytes"; + const hash = createHash("md5").update(content).digest("hex"); + const relative = `${NAME}/owned.md`; + const machine = manifest(); + machine.addPlugin("cursor", plugin({ files: { [relative]: hash, "../outside.md": hash } })); + const before = machine.toJSON(); + const base = join(homedir(), ".cursor", "plugins", "local"); + const fs = new InMemoryFileAdapter({ + [join(base, relative)]: content, + [join(base, "../outside.md")]: content, + }); + const userRepo = new InMemoryManifestRepository(machine); + const remove = new PluginRemoveUseCase( + fs, + new InMemoryManifestRepository(manifest(), ROOT), + new CapturingLogger(), + new Map(), + new Map(), + undefined, + undefined, + userRepo + ); + + await expect( + remove.execute({ pluginName: NAME, toolIds: ["cursor"], projectRoot: ROOT, scope: "user" }) + ).rejects.toThrow( + `Refusing partial removal of user-scope plugin '${NAME}': a tracked file escaped its boundary.` + ); + + expect(fs.getFile(join(base, relative))).toBe(content); + expect(fs.getFile(join(base, "../outside.md"))).toBe(content); + expect(machine.toJSON()).toEqual(before); + expect(userRepo.saveCount).toBe(0); + }); + + it("refuses unproven project hooks before deleting any tracked plugin file", async () => { + const project = manifest(); + project.addPlugin("cursor", plugin({ scope: "project", files: { "owned.md": "digest" } })); + const before = project.toJSON(); + const script = join(ROOT, ".cursor", "hooks", NAME, "edited.sh"); + const file = join(ROOT, "owned.md"); + const fs = new InMemoryFileAdapter({ + [script]: "unproven hook bytes", + [file]: "installed plugin bytes", + }); + const repo = new InMemoryManifestRepository(project, ROOT); + const remove = new PluginRemoveUseCase(fs, repo, new CapturingLogger(), new Map()); + + await expect( + remove.execute({ pluginName: NAME, toolIds: ["cursor"], projectRoot: ROOT }) + ).rejects.toThrow(/unproven legacy install digest/); + + expect(fs.getFile(file)).toBe("installed plugin bytes"); + expect(fs.getFile(script)).toBe("unproven hook bytes"); + expect(project.toJSON()).toEqual(before); + expect(repo.saveCount).toBe(0); + }); + + it("preserves a scope-aware host ref when no activator confirmed its removal", async () => { + const project = manifest(); + project.addPlugin("claude", plugin({ scope: "project", marketplace: "local-alias" })); + const ref = `${NAME}@host-catalogue`; + const registrations = { + binary: "claude", + marketplaces: [{ alias: "local-alias", hostName: "host-catalogue" }], + pluginRefs: [ref, "other-plugin@host-catalogue"], + pluginClaims: [{ ref, dependents: [ROOT, OTHER] }], + }; + project.setNativeRegistrations("claude", registrations); + const machine = manifest(); + machine.setNativeRegistrations("claude", registrations); + const repo = new InMemoryManifestRepository(project, ROOT); + const userRepo = new InMemoryManifestRepository(machine); + const remove = new PluginRemoveUseCase( + new InMemoryFileAdapter(), + repo, + new CapturingLogger(), + new Map(), + new Map(), + undefined, + undefined, + userRepo + ); + + await remove.execute({ pluginName: NAME, toolIds: ["claude"], projectRoot: ROOT }); + + expect(project.getNativeRegistrations("claude")?.pluginRefs).toEqual([ + ref, + "other-plugin@host-catalogue", + ]); + expect(machine.getNativeRegistrations("claude")?.pluginRefs).toEqual([ + ref, + "other-plugin@host-catalogue", + ]); + expect(machine.getNativeRegistrations("claude")?.pluginClaims).toEqual([ + { ref, dependents: [OTHER] }, + ]); + expect(project.getPlugins("claude")).toEqual([]); + expect(userRepo.saveCount).toBe(1); + expect(repo.saveCount).toBe(1); + }); }); describe("native removal preserves unconfirmed refs and addresses the recorded catalogue", () => {