From e6e457d55609056406d45ff576b69c053a94c6bc Mon Sep 17 00:00:00 2001 From: Rostislav Rotaru Date: Sat, 14 Nov 2026 15:17:59 -0800 Subject: [PATCH 1/4] Run each suite under its own hang limit, print its slowest tests, and time it - build.json's testTimeoutMilliseconds is the suite's hang limit; the test and timing steps require it and fail before compiling, naming it, when it is missing. - The test step writes the run's test times to _build/test-results.json and prints the 10 slowest, even with --skip-test-details. - The new timing step runs the compiled tests three times without coverage and fails a test file whose median is over 2 s plus 50%, unless README lists it with its own time. - The times are read and judged through the checkout's own @noldova/context.testing, loaded as the coverage report already loads it. - context's packages move to 0.0.8, which has the run limit, results file and timing classes. Issue #161 Agent: DEV1 (developer) Agent-Model: Anthropic Opus 5.5, Medium --- package-lock.json | 62 ++--- package.json | 6 +- src/noldova.tools.pipelines.github/README.md | 3 +- .../package.json | 4 +- .../src/services/pipelines-program.ts | 4 +- .../tests/bin/pipelines.test.ts | 2 +- .../tests/services/pipelines-program.test.ts | 2 +- src/noldova.tools.pipelines/README.md | 19 +- src/noldova.tools.pipelines/package.json | 4 +- .../src/api/index.d.ts | 104 +++++-- src/noldova.tools.pipelines/src/api/index.ts | 1 + .../src/interfaces/i-testing-module.ts | 15 + .../src/models/build-config.ts | 10 +- src/noldova.tools.pipelines/src/resources.ts | 42 +++ .../src/services/coverage-entry-point.ts | 22 +- .../src/services/test-step.ts | 16 +- .../src/services/test-times-entry-point.ts | 43 +++ .../src/services/testing-module-loader.ts | 35 +++ .../src/services/timing-step.ts | 102 +++++++ .../tests/api/index.test.ts | 2 +- .../tests/models/build-config.test.ts | 21 ++ .../tests/services/test-step.test.ts | 56 +++- .../tests/services/timing-step.test.ts | 261 ++++++++++++++++++ 23 files changed, 742 insertions(+), 94 deletions(-) create mode 100644 src/noldova.tools.pipelines/src/services/test-times-entry-point.ts create mode 100644 src/noldova.tools.pipelines/src/services/testing-module-loader.ts create mode 100644 src/noldova.tools.pipelines/src/services/timing-step.ts create mode 100644 src/noldova.tools.pipelines/tests/services/timing-step.test.ts diff --git a/package-lock.json b/package-lock.json index b8d6feb..903eebc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,9 +7,9 @@ "name": "@noldova/tools", "license": "MIT", "devDependencies": { - "@noldova/context.exceptions": "0.0.7", - "@noldova/context.json": "0.0.7", - "@noldova/context.testing": "0.0.7", + "@noldova/context.exceptions": "0.0.8", + "@noldova/context.json": "0.0.8", + "@noldova/context.testing": "0.0.8", "@types/node": "24.19.2", "typescript": "7.0.2" }, @@ -19,63 +19,63 @@ } }, "node_modules/@noldova/context.collections": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/@noldova/context.collections/-/context.collections-0.0.7.tgz", - "integrity": "sha512-AH3QCTqPSJ7fH1StHnfQs+6CXlqGcgKI2shzcsjpJGd2rk1Wyn9FXzAHu0uXXGR3E0UYhlnCoyIWeKqM7vtpcA==", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@noldova/context.collections/-/context.collections-0.0.8.tgz", + "integrity": "sha512-fStNqT0sY3hSAWUCynkp7nm9aSOguZSyFULbAxrHwxueeNGEO6bc5GkDT4hBeGR1DUKNM0AV4wW1ovz7GScUrg==", "dev": true, "license": "MIT" }, "node_modules/@noldova/context.core": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/@noldova/context.core/-/context.core-0.0.7.tgz", - "integrity": "sha512-gi5mlQB3rc0FmCcLpHVkE8p9jPNtge7PF2jlms6uVw1Ev9CSyHDBRgRfh6W7w02HbJ7JQx+IJYqnn8mbfc5+CA==", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@noldova/context.core/-/context.core-0.0.8.tgz", + "integrity": "sha512-W0ApMPnhUCNZa8a7ESKotcHnnuxksJfvXYmdvxgHfLwYM0oUfpNMRvIpbPPVda9OVvA1xjP0m3hLI/FEo+JYkA==", "dev": true, "license": "MIT" }, "node_modules/@noldova/context.exceptions": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/@noldova/context.exceptions/-/context.exceptions-0.0.7.tgz", - "integrity": "sha512-AA5HuwxKdrpF5/C/Nv1yPVwTfJOR46Uk5t7jDacU8xiAqBo15tJv61rPBDgXpymWPZinWQ6W13YlRzKC6rylcg==", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@noldova/context.exceptions/-/context.exceptions-0.0.8.tgz", + "integrity": "sha512-h0RS5PSpm+N4MCp7xl0fAJcpRDmdwLsWQnj3Fp560MWCDwMWiFNl/PC+OGgrqbTmoMxw6+ln6RravDG9Nmy00A==", "dev": true, "license": "MIT", "dependencies": { - "@noldova/context.core": "0.0.7" + "@noldova/context.core": "0.0.8" } }, "node_modules/@noldova/context.json": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/@noldova/context.json/-/context.json-0.0.7.tgz", - "integrity": "sha512-Wd09/7PvHLGiVD458Pk2zIl/xPAEQ3W7b1wzH6/1nsUtddVf+STzfz3wBCsQiLaeANent3v0Fo+1ASgydH8L1w==", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@noldova/context.json/-/context.json-0.0.8.tgz", + "integrity": "sha512-A0jqSTw6izkdKW8mTWHxY6QxmgWvdqtCZZuMmAAqtsbaeRPOn9iItV99SubMxhd0nNbOhreryXkuRFRpjt+TpQ==", "dev": true, "license": "MIT", "dependencies": { - "@noldova/context.core": "0.0.7", - "@noldova/context.exceptions": "0.0.7" + "@noldova/context.core": "0.0.8", + "@noldova/context.exceptions": "0.0.8" } }, "node_modules/@noldova/context.testing": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/@noldova/context.testing/-/context.testing-0.0.7.tgz", - "integrity": "sha512-svyzZ072eEqKOcneRs7TIxfjCSVb0gCOIhjIuEwIOIpjLc/8WozfToo8bW443/SVZumhy7YkmhRMT8uAzN+D8g==", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@noldova/context.testing/-/context.testing-0.0.8.tgz", + "integrity": "sha512-g4Sn0J7QX0nHkVHXg1diGWsnPpQVl0KzsC8sztTUjauZom4vnedtQl1CsVG6SgyLrRz+zp3KP4fGdLtAvjBliQ==", "dev": true, "license": "MIT", "dependencies": { - "@noldova/context.collections": "0.0.7", - "@noldova/context.core": "0.0.7", - "@noldova/context.exceptions": "0.0.7", - "@noldova/context.text": "0.0.7" + "@noldova/context.collections": "0.0.8", + "@noldova/context.core": "0.0.8", + "@noldova/context.exceptions": "0.0.8", + "@noldova/context.text": "0.0.8" } }, "node_modules/@noldova/context.text": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/@noldova/context.text/-/context.text-0.0.7.tgz", - "integrity": "sha512-3EhgMHiGZjACIOa/bi7BZQzWQ8lT6gNXF1q7aRFoCAQdEJokO5/1dSBfgeAoOLz5I+csL7k1bK7ulvvc7Ue/4Q==", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@noldova/context.text/-/context.text-0.0.8.tgz", + "integrity": "sha512-Wp1uZ/5mTkZJGafRA+eFAnmvyk7y6kv7FpJ6UUzKFF82LSOxICeLlwBwINTweoMdttXeLuE+BVHPVweBU8dMyA==", "dev": true, "license": "MIT", "dependencies": { - "@noldova/context.collections": "0.0.7", - "@noldova/context.core": "0.0.7", - "@noldova/context.exceptions": "0.0.7" + "@noldova/context.collections": "0.0.8", + "@noldova/context.core": "0.0.8", + "@noldova/context.exceptions": "0.0.8" } }, "node_modules/@types/node": { diff --git a/package.json b/package.json index 2708e2d..9d8267c 100644 --- a/package.json +++ b/package.json @@ -14,9 +14,9 @@ "pipelines": "node -e \"if (!require('fs').existsSync('.pipelines/node_modules/@noldova/tools.pipelines.github')) { console.error('.pipelines holds no build tool yet; npm run build installs it.'); process.exit(1); }\" && node .pipelines/node_modules/@noldova/tools.pipelines.github/bin/pipelines.js" }, "devDependencies": { - "@noldova/context.exceptions": "0.0.7", - "@noldova/context.json": "0.0.7", - "@noldova/context.testing": "0.0.7", + "@noldova/context.exceptions": "0.0.8", + "@noldova/context.json": "0.0.8", + "@noldova/context.testing": "0.0.8", "@types/node": "24.19.2", "typescript": "7.0.2" } diff --git a/src/noldova.tools.pipelines.github/README.md b/src/noldova.tools.pipelines.github/README.md index 4ac0739..2790993 100644 --- a/src/noldova.tools.pipelines.github/README.md +++ b/src/noldova.tools.pipelines.github/README.md @@ -1,7 +1,7 @@ # @noldova/tools.pipelines.github Runs the steps of `@noldova/tools.pipelines` on GitHub Actions. -This version holds `pipelines`, the command line a workflow calls, with the source, archive, publish, nightly, prune, tag, toolchain, line-endings, unchanged, reproducible, dependents, pins, installed, build and test steps, and the prune, tag, dependents and pins steps' access to GitHub. +This version holds `pipelines`, the command line a workflow calls, with the source, archive, publish, nightly, prune, tag, toolchain, line-endings, unchanged, reproducible, dependents, pins, installed, build, test and timing steps, and the prune, tag, dependents and pins steps' access to GitHub. ## Install @@ -42,6 +42,7 @@ A job installs the package at an exact version, without install scripts, and cal | `installed` | `[]` | | `build` | `[--version ] [...]`, or `--list-copies` | | `test` | `[--skip-test-details] [--skip-coverage-details] [...]` | +| `timing` | none | `@noldova/tools.pipelines`'s README says what each step checks and does. The reproducible, dependents and pins steps make their temporary folders in `RUNNER_TEMP`, which the runner empties after each job, cancelled ones included. diff --git a/src/noldova.tools.pipelines.github/package.json b/src/noldova.tools.pipelines.github/package.json index 67de914..3fb7a73 100644 --- a/src/noldova.tools.pipelines.github/package.json +++ b/src/noldova.tools.pipelines.github/package.json @@ -22,8 +22,8 @@ "registry": "https://registry.npmjs.org/" }, "dependencies": { - "@noldova/context.exceptions": "0.0.7", - "@noldova/context.json": "0.0.7", + "@noldova/context.exceptions": "0.0.8", + "@noldova/context.json": "0.0.8", "@noldova/tools.pipelines": "__VERSION__" } } diff --git a/src/noldova.tools.pipelines.github/src/services/pipelines-program.ts b/src/noldova.tools.pipelines.github/src/services/pipelines-program.ts index fb8be0f..76434b3 100644 --- a/src/noldova.tools.pipelines.github/src/services/pipelines-program.ts +++ b/src/noldova.tools.pipelines.github/src/services/pipelines-program.ts @@ -6,7 +6,7 @@ * LICENSE file in the root directory of this source tree. */ -import { ArchiveStep, BuildStep, CommandLine, DependentsStep, Git, InstalledStep, type IStepOutput, LineEndingsStep, NightlyStep, Npm, PinsStep, ProcessRunner, PruneStep, PublishStep, RelockStep, ReproducibleStep, SourceStep, TagStep, Tar, TestStep, ToolchainStep, TypeScript, UnchangedStep } from "@noldova/tools.pipelines"; +import { ArchiveStep, BuildStep, CommandLine, DependentsStep, Git, InstalledStep, type IStepOutput, LineEndingsStep, NightlyStep, Npm, PinsStep, ProcessRunner, PruneStep, PublishStep, RelockStep, ReproducibleStep, SourceStep, TagStep, Tar, TestStep, TimingStep, ToolchainStep, TypeScript, UnchangedStep } from "@noldova/tools.pipelines"; import { Resources } from "../resources.js"; import { GitHubPackages } from "./github-packages.js"; @@ -26,7 +26,7 @@ export class PipelinesProgram { const npm = new Npm(processRunner, process.cwd()); const archiveStep = new ArchiveStep(git, new Tar(processRunner), npm); const typeScript = new TypeScript(processRunner, process.cwd()); - const steps = [new SourceStep(git), archiveStep, new PublishStep(archiveStep, npm), new NightlyStep(git, archiveStep, npm), new PruneStep(git, archiveStep, new GitHubPackages(process.env)), new TagStep(new GitHubTags(process.env)), new ToolchainStep(npm, process.version), new LineEndingsStep(git), new UnchangedStep(git), new ReproducibleStep(git, npm, process.env[Resources.runnerTemporaryVariable]), new DependentsStep(archiveStep, new GitHubUpdates(process.env, processRunner, undefined, undefined, undefined, this.output), processRunner, npm, process.env[Resources.runnerTemporaryVariable], this.output), new PinsStep(git, new GitHubUpdates(process.env, processRunner, undefined, undefined, undefined, this.output), processRunner, process.env[Resources.runnerTemporaryVariable], this.output), new RelockStep(processRunner, npm, process.cwd()), new InstalledStep(process.cwd(), this.output), new BuildStep(git, npm, typeScript, processRunner, this.output), new TestStep(typeScript, processRunner, this.output)]; + const steps = [new SourceStep(git), archiveStep, new PublishStep(archiveStep, npm), new NightlyStep(git, archiveStep, npm), new PruneStep(git, archiveStep, new GitHubPackages(process.env)), new TagStep(new GitHubTags(process.env)), new ToolchainStep(npm, process.version), new LineEndingsStep(git), new UnchangedStep(git), new ReproducibleStep(git, npm, process.env[Resources.runnerTemporaryVariable]), new DependentsStep(archiveStep, new GitHubUpdates(process.env, processRunner, undefined, undefined, undefined, this.output), processRunner, npm, process.env[Resources.runnerTemporaryVariable], this.output), new PinsStep(git, new GitHubUpdates(process.env, processRunner, undefined, undefined, undefined, this.output), processRunner, process.env[Resources.runnerTemporaryVariable], this.output), new RelockStep(processRunner, npm, process.cwd()), new InstalledStep(process.cwd(), this.output), new BuildStep(git, npm, typeScript, processRunner, this.output), new TestStep(typeScript, processRunner, this.output), new TimingStep(process.cwd(), processRunner, this.output)]; return new CommandLine(Resources.programName, steps, this.output).runAsync(values); } diff --git a/src/noldova.tools.pipelines.github/tests/bin/pipelines.test.ts b/src/noldova.tools.pipelines.github/tests/bin/pipelines.test.ts index 5512441..e11c686 100644 --- a/src/noldova.tools.pipelines.github/tests/bin/pipelines.test.ts +++ b/src/noldova.tools.pipelines.github/tests/bin/pipelines.test.ts @@ -41,7 +41,7 @@ export class PipelinesBinTests { Assert.areEqual(1, run.exitCode); Assert.areEqual("", run.standardOutput); Assert.areEqual( - "::error::Usage: pipelines source | archive | publish | nightly | prune | tag | toolchain [] | line-endings | unchanged | reproducible | dependents | pins | relock [=...] | installed [] | build [--version ] [...], or --list-copies | test [--skip-test-details] [--skip-coverage-details] [...]\n", + "::error::Usage: pipelines source | archive | publish | nightly | prune | tag | toolchain [] | line-endings | unchanged | reproducible | dependents | pins | relock [=...] | installed [] | build [--version ] [...], or --list-copies | test [--skip-test-details] [--skip-coverage-details] [...] | timing\n", run.standardError); } diff --git a/src/noldova.tools.pipelines.github/tests/services/pipelines-program.test.ts b/src/noldova.tools.pipelines.github/tests/services/pipelines-program.test.ts index 53935b1..9d4d6f9 100644 --- a/src/noldova.tools.pipelines.github/tests/services/pipelines-program.test.ts +++ b/src/noldova.tools.pipelines.github/tests/services/pipelines-program.test.ts @@ -43,7 +43,7 @@ export class PipelinesProgramTests { Assert.areEqual(1, exitCode); Assert.areEqual( - "[\"failure: Usage: pipelines source | archive | publish | nightly | prune | tag | toolchain [] | line-endings | unchanged | reproducible | dependents | pins | relock [=...] | installed [] | build [--version ] [...], or --list-copies | test [--skip-test-details] [--skip-coverage-details] [...]\"]", + "[\"failure: Usage: pipelines source | archive | publish | nightly | prune | tag | toolchain [] | line-endings | unchanged | reproducible | dependents | pins | relock [=...] | installed [] | build [--version ] [...], or --list-copies | test [--skip-test-details] [--skip-coverage-details] [...] | timing\"]", JSON.stringify(output.lines)); } diff --git a/src/noldova.tools.pipelines/README.md b/src/noldova.tools.pipelines/README.md index 5c5ecad..d73bc74 100644 --- a/src/noldova.tools.pipelines/README.md +++ b/src/noldova.tools.pipelines/README.md @@ -357,12 +357,13 @@ The declaration file `api/index.d.ts` documents every member with its contract a ### Building and testing a repository -The `installed`, `build` and `test` steps build, pack and test the packages of a repository, so every repository runs the same build. +The `installed`, `build`, `test` and `timing` steps build, pack, test and time the packages of a repository, so every repository runs the same build. A repository lists its packages, in the order they build, in `build.json` at its root: ```json { "scope": "@noldova", + "testTimeoutMilliseconds": 30000, "packages": [ { "name": "noldova.example" }, { "name": "noldova.example.parsers", "path": "parsers/noldova.example.parsers", "pre": "src/parsers/noldova.example.parsers/scripts/verify.ts" } @@ -371,6 +372,7 @@ A repository lists its packages, in the order they build, in `build.json` at its ``` - `scope` is the npm scope the packages publish under, `@noldova` when omitted. +- `testTimeoutMilliseconds` is the suite's hang limit, a positive whole number of milliseconds, which the `test` and `timing` steps require: at least three times the slowest test on any target, as the test step's slowest tests show it in the repository's runs, with that measurement in its README's Checks. A test that reaches it fails, named with what it waited for. - `name` is the package's folder name: `noldova.` and segments of lowercase letters and digits joined by dots, with no hyphen, so a several-word segment joins its words with nothing between them, such as `noldova.example.filewatcher`; the package's npm name is the scope, `/` and the name after `noldova.`, such as `@noldova/example`. - `path` is the package's folder under `src/`, whose last segment is the name; it is the name when omitted. - `pre`, `compile` and `post` are optional scripts, paths inside the checkout that Node.js runs with the package's source folder and build folder as its values: `pre` before the compile, `compile` instead of the TypeScript compile, and `post` after it. @@ -382,7 +384,8 @@ A repository's `package.json` runs the steps through the package's `pipelines` c ```json "scripts": { "build": "(pipelines installed || npm ci --ignore-scripts) && pipelines build", - "test": "pipelines test" + "test": "pipelines test", + "test:timing": "pipelines timing" } ``` @@ -412,13 +415,21 @@ It backs up `package.json` and `package-lock.json` in `_build/restore/` before i #### The test step -`test [--skip-test-details] [--skip-coverage-details] [...]` compiles each package's `tests/tsconfig.json`, runs the tests through the repository's `@noldova/context.testing` with V8 coverage in `_build/coverage`, and requires complete coverage of the packages' `src/` folders. +`test [--skip-test-details] [--skip-coverage-details] [...]` compiles each package's `tests/tsconfig.json`, runs the tests through the repository's `@noldova/context.testing` under `build.json`'s `testTimeoutMilliseconds`, with V8 coverage in `_build/coverage`, and requires complete coverage of the packages' `src/` folders. +The run writes each test's time to `_build/test-results.json`, and the step then prints the 10 slowest tests with their times, with or without `--skip-test-details`, and fails when it can't read them. It writes `Compiling the tests of ...` for each, and ends the line with ` Done.`, or ` Failed.` before the compiler's errors. - Run the build first: the tests use the installed packages. - A filter runs the tests that match a package, file, class, `ClassName.methodName` substring or exact `category:name` identity, and skips the coverage gate. - `--skip-test-details` hides the passing tests' lines and `--skip-coverage-details` the fully covered files' rows; failures and totals always show. -- The tests and the coverage report each run in a Node.js process of their own, so the repository's packages never load beside the step's. +- The tests, their times and the coverage report each run in a Node.js process of their own, so the repository's packages never load beside the step's. + +#### The timing step + +`timing` checks the test times, apart from the test step since its pass condition is a time, so it runs on a test machine rather than in CI. +Run it after the test step, which compiles the tests: it runs them three times without coverage, writes `Timing run of 3...` for each, and judges each test file's time, the sum of its tests' durations, through the repository's `@noldova/context.testing`. +It fails a file whose median over the runs is over 2 seconds plus a margin of 50%, since a busy machine slows every file and the median of three runs drops one slow run but not a slower machine. +A file slower than that budget is listed in the repository's README, after the line `Test files slower than the timing check's budget:`, as a table row such as ``| `noldova.example/services/a.test.js` | 6900 ms |``; its median may exceed its own time by the same margin, and a listed file within the budget is named as stale, failing nothing. ## License diff --git a/src/noldova.tools.pipelines/package.json b/src/noldova.tools.pipelines/package.json index 14ff454..5cc0e42 100644 --- a/src/noldova.tools.pipelines/package.json +++ b/src/noldova.tools.pipelines/package.json @@ -19,7 +19,7 @@ "registry": "https://registry.npmjs.org/" }, "dependencies": { - "@noldova/context.exceptions": "0.0.7", - "@noldova/context.json": "0.0.7" + "@noldova/context.exceptions": "0.0.8", + "@noldova/context.json": "0.0.8" } } diff --git a/src/noldova.tools.pipelines/src/api/index.d.ts b/src/noldova.tools.pipelines/src/api/index.d.ts index 295afd1..9b8a34b 100644 --- a/src/noldova.tools.pipelines/src/api/index.d.ts +++ b/src/noldova.tools.pipelines/src/api/index.d.ts @@ -594,11 +594,19 @@ export declare class BuildConfig { */ public readonly packages: readonly BuildPackage[]; + /** + * The suite's hang limit in milliseconds, `build.json`'s + * `testTimeoutMilliseconds`, which {@link TestStep} and + * {@link TimingStep} require; undefined when the file sets none. + */ + public readonly testTimeoutMilliseconds: number | undefined; + /** * Creates the configuration. * * @param scope The npm scope the packages publish under. * @param packages The packages, in build order. + * @param testTimeoutMilliseconds The suite's hang limit in milliseconds, or undefined for none. * @example * ```ts * import { BuildConfig, BuildPackage } from "@noldova/tools.pipelines"; @@ -606,7 +614,7 @@ export declare class BuildConfig { * const config = new BuildConfig("@noldova", [new BuildPackage("noldova.tools.pipelines")]); * ``` */ - public constructor(scope: string, packages: readonly BuildPackage[]); + public constructor(scope: string, packages: readonly BuildPackage[], testTimeoutMilliseconds?: number | undefined); /** * The packages' npm names, such as `@noldova/tools.pipelines`. @@ -619,9 +627,9 @@ export declare class BuildConfig { * @param directory The checkout's root. * @returns The configuration. * @throws {@link BuildException} when the file is missing or names a scope - * that is not an npm scope, no package, a package twice, a name that is not a package folder name, or a - * path or a script outside the checkout; `JsonException` from - * `@noldova/context.json` when it is not JSON or lacks a field. + * that is not an npm scope, no package, a package twice, a name that is not a package folder name, a + * path or a script outside the checkout, or a `testTimeoutMilliseconds` that is not positive; `JsonException` from + * `@noldova/context.json` when it is not JSON, lacks a field, or has a `testTimeoutMilliseconds` that is not an integer. * @example * ```ts * import { BuildConfig } from "@noldova/tools.pipelines"; @@ -637,9 +645,9 @@ export declare class BuildConfig { * @param text The file's text. * @returns The configuration. * @throws {@link BuildException} when the text names a scope that is not - * an npm scope, no package, a package twice, a name that is not a package folder name, or a path or a - * script outside the checkout; `JsonException` from - * `@noldova/context.json` when it is not JSON or lacks a field. + * an npm scope, no package, a package twice, a name that is not a package folder name, a path or a + * script outside the checkout, or a `testTimeoutMilliseconds` that is not positive; `JsonException` from + * `@noldova/context.json` when it is not JSON, lacks a field, or has a `testTimeoutMilliseconds` that is not an integer. * @example * ```ts * import { BuildConfig } from "@noldova/tools.pipelines"; @@ -3171,11 +3179,14 @@ export declare class BuildStep implements IStep { * `test [--skip-test-details] [--skip-coverage-details] [...]`. It * compiles each package's `tests/tsconfig.json` into `_build/tests/`, * runs them through the checkout's `@noldova/context.testing` with V8 - * coverage, and requires complete coverage of the packages' source; a run - * with filters skips the coverage gate. The tests and the coverage report - * each run in a Node.js process of their own, so the checkout's packages - * never load beside the step's. Run the build first: the tests use the - * installed packages. + * coverage, under the hang limit `build.json`'s `testTimeoutMilliseconds` + * sets, and requires complete coverage of the packages' source; a run with + * filters skips the coverage gate. The run writes each test's time to + * `_build/test-results.json`, and the step prints the 10 slowest, even with + * `--skip-test-details`. The tests, their times and the coverage report each + * run in a Node.js process of their own, so the checkout's packages never + * load beside the step's. Run the build first: the tests use the installed + * packages. */ export declare class TestStep implements IStep { /** @@ -3216,14 +3227,75 @@ export declare class TestStep implements IStep { * `ClassName.methodName` substring or exact `category:name` identity. * @returns A promise of the result: succeeded, such as * `The tests passed, and coverage is complete.`, or failed for an unknown - * option, a `build.json` that is not valid, no package with tests, a - * compile that fails, failed tests, incomplete coverage, or a coverage - * report that cannot run, such as without `@noldova/context.testing` + * option, a `build.json` that is not valid or sets no + * `testTimeoutMilliseconds`, no package with tests, a compile that fails, + * failed tests, test times that cannot be read, incomplete coverage, or a + * coverage report that cannot run, such as without + * `@noldova/context.testing` installed. It rejects with + * {@link ProcessException} if a program cannot start. + * @example + * ```ts + * (await step.runAsync(["--skip-test-details"])).message; // "The tests passed, and coverage is complete." + * ``` + */ + public runAsync(values: readonly string[]): Promise; +} + +/** + * The step that checks a checkout's test times: `timing`, run after the + * test step, which compiles the tests. It runs the compiled tests three + * times without coverage, under `build.json`'s `testTimeoutMilliseconds`, + * and through the checkout's `@noldova/context.testing` judges each test + * file's time, the sum of its tests' durations: it fails a file whose median + * over the runs exceeds 2 seconds plus a margin of 50%, unless the + * checkout's README lists the file with a time of its own, after the line + * `Test files slower than the timing check's budget:`, which the file's + * median may exceed by the same margin. It names a listed file within the + * budget as stale, failing nothing. Its pass condition is a time, so it + * runs apart from the test step, on a test machine. + */ +export declare class TimingStep implements IStep { + /** + * The step's name, `"timing"`. + */ + public readonly name: string; + + /** + * The values the step takes: `""`, none. + */ + public readonly usage: string; + + /** + * Creates the step. + * + * @param workingDirectory The checkout's root. + * @param processRunner What runs the tests and judges their times. + * @param output Where the step writes its progress. + * @example + * ```ts + * import { ProcessRunner, TimingStep, type IStepOutput } from "@noldova/tools.pipelines"; + * + * declare const output: IStepOutput; + * const step = new TimingStep(process.cwd(), new ProcessRunner(), output); + * step.name; // "timing" + * ``` + */ + public constructor(workingDirectory: string, processRunner: ProcessRunner, output: IStepOutput); + + /** + * Runs the tests three times and judges their times. + * + * @param values None. + * @returns A promise of the result: succeeded, `Every test file is within + * its time.`, or failed for a value given, a `build.json` that is not + * valid or sets no `testTimeoutMilliseconds`, no package with tests, tests + * not compiled, a run that fails, a test file over its time, or test times + * that cannot be read, such as without `@noldova/context.testing` * installed. It rejects with {@link ProcessException} if a program cannot * start. * @example * ```ts - * (await step.runAsync(["--skip-test-details"])).message; // "The tests passed, and coverage is complete." + * (await step.runAsync([])).message; // "Every test file is within its time." * ``` */ public runAsync(values: readonly string[]): Promise; diff --git a/src/noldova.tools.pipelines/src/api/index.ts b/src/noldova.tools.pipelines/src/api/index.ts index 882b7a2..545a578 100644 --- a/src/noldova.tools.pipelines/src/api/index.ts +++ b/src/noldova.tools.pipelines/src/api/index.ts @@ -43,6 +43,7 @@ export { SourceStep } from "../services/source-step.js"; export { TagStep } from "../services/tag-step.js"; export { Tar } from "../services/tar.js"; export { TestStep } from "../services/test-step.js"; +export { TimingStep } from "../services/timing-step.js"; export { ToolchainStep } from "../services/toolchain-step.js"; export { TypeScript } from "../services/type-script.js"; export { UnchangedStep } from "../services/unchanged-step.js"; diff --git a/src/noldova.tools.pipelines/src/interfaces/i-testing-module.ts b/src/noldova.tools.pipelines/src/interfaces/i-testing-module.ts index b0721fb..be77946 100644 --- a/src/noldova.tools.pipelines/src/interfaces/i-testing-module.ts +++ b/src/noldova.tools.pipelines/src/interfaces/i-testing-module.ts @@ -10,8 +10,23 @@ export interface ICoverage { readonly isComplete: boolean; } +export interface ITestFileTiming { + readonly isOver: boolean; + readonly isListed: boolean; + format(): string; +} + export interface ITestingModule { readonly CoverageProject: new (name: string, packageFolder: string, sourceFolder: string) => unknown; readonly CoverageAnalyzer: new () => { analyzeAsync(coverageFolder: string, projects: readonly unknown[]): Promise }; readonly CoverageReportWriter: new () => { formatLines(coverage: ICoverage, skipDetails: boolean): readonly string[] }; + readonly TestTimeRecord: { + parseAll(text: string): readonly unknown[]; + formatSlowest(records: readonly unknown[], count: number): readonly string[]; + }; + readonly TestTimingCheck: { + new (budgetMilliseconds: number, margin: number, listed: ReadonlyMap): { judge(runs: readonly (readonly unknown[])[]): readonly ITestFileTiming[] }; + readListed(markdown: string): ReadonlyMap; + formatSummary(timings: readonly ITestFileTiming[], runs: number): string; + }; } diff --git a/src/noldova.tools.pipelines/src/models/build-config.ts b/src/noldova.tools.pipelines/src/models/build-config.ts index 8794967..b5fddd6 100644 --- a/src/noldova.tools.pipelines/src/models/build-config.ts +++ b/src/noldova.tools.pipelines/src/models/build-config.ts @@ -18,10 +18,12 @@ import { BuildPackage } from "./build-package.js"; export class BuildConfig { public readonly scope: string; public readonly packages: readonly BuildPackage[]; + public readonly testTimeoutMilliseconds: number | undefined; - public constructor(scope: string, packages: readonly BuildPackage[]) { + public constructor(scope: string, packages: readonly BuildPackage[], testTimeoutMilliseconds: number | undefined = undefined) { this.scope = scope; this.packages = packages; + this.testTimeoutMilliseconds = testTimeoutMilliseconds; } public get packageNames(): readonly string[] { @@ -55,7 +57,11 @@ export class BuildConfig { packages.push(buildPackage); } - return new BuildConfig(scope, packages); + const testTimeoutMilliseconds = reader.hasField(Resources.testTimeoutField) ? reader.readInteger(Resources.testTimeoutField) : undefined; + if (testTimeoutMilliseconds !== undefined && testTimeoutMilliseconds <= 0) + throw new BuildException(Resources.formatBuildConfigInvalid(Resources.formatTestTimeoutInvalid(testTimeoutMilliseconds))); + + return new BuildConfig(scope, packages, testTimeoutMilliseconds); } private static readPackage(entry: JsonReader, scope: string): BuildPackage { diff --git a/src/noldova.tools.pipelines/src/resources.ts b/src/noldova.tools.pipelines/src/resources.ts index e41448d..fab2f22 100644 --- a/src/noldova.tools.pipelines/src/resources.ts +++ b/src/noldova.tools.pipelines/src/resources.ts @@ -257,12 +257,34 @@ export class Resources { public static readonly testFiltersVariable: string = "CONTEXT_TEST_FILTERS"; public static readonly testOptions: readonly string[] = ["--skip-test-details", "--skip-coverage-details"]; public static readonly testStepName: string = "test"; + public static readonly testTimeoutField: string = "testTimeoutMilliseconds"; + public static readonly testTimeoutMissing: string = + "build.json sets no testTimeoutMilliseconds, the suite's hang limit: at least three times its slowest test on any target, with the measurement in README's Checks."; + public static readonly testTimeoutVariable: string = "CONTEXT_TEST_TIMEOUT_MILLISECONDS"; public static readonly testStepUsage: string = "[--skip-test-details] [--skip-coverage-details] [...]"; public static readonly testingMissing: string = "@noldova/context.testing is not installed; the repository's devDependencies name it."; public static readonly testingPackageName: string = "@noldova/context.testing"; public static readonly testsFolder: string = "_build/tests"; public static readonly testsMissing: string = "No package has tests."; public static readonly testsPassed: string = "The tests passed, and coverage is complete."; + public static readonly testResultsFile: string = "_build/test-results.json"; + public static readonly testResultsVariable: string = "CONTEXT_TEST_RESULTS_FILE"; + public static readonly testTimesEntryPointPath: string = "./test-times-entry-point.js"; + public static readonly slowestMode: string = "slowest"; + public static readonly judgeMode: string = "judge"; + public static readonly slowestTestCount: number = 10; + public static readonly timesUnreadExitCode: number = 2; + public static readonly timingStepName: string = "timing"; + public static readonly timingStepUsage: string = ""; + public static readonly timingFolder: string = "_build/timing"; + public static readonly timingRuns: number = 3; + // A test file's time is the sum of its tests' durations; the margin is wide since a busy machine slows every file, and the median of three + // runs drops one slow run but not a slower machine. + public static readonly timingBudgetMilliseconds: number = 2_000; + public static readonly timingMargin: number = 0.5; + public static readonly timingNotBuilt: string = "The tests are not compiled; run the test step first."; + public static readonly timingPassed: string = "Every test file is within its time."; + public static readonly timingOver: string = "A test file is over its time."; public static readonly testsProjectPath: string = "tests/tsconfig.json"; public static readonly typeScriptCompileValues: readonly string[] = ["--pretty", "--project"]; public static readonly typeScriptCompilerPath: string = "bin/tsc"; @@ -991,6 +1013,10 @@ export class Resources { return `build.json is invalid: ${problem}.`; } + public static formatTestTimeoutInvalid(milliseconds: number): string { + return `testTimeoutMilliseconds is ${milliseconds}, not a positive number of milliseconds`; + } + public static formatBuildPackageDuplicated(name: string): string { return `it names ${name} more than once`; } @@ -1086,6 +1112,22 @@ export class Resources { return `Compiling the tests of ${name}...`; } + public static formatTimesUnread(exitCode: number | null): string { + return `The test times were not read: their report exited with code ${exitCode}.`; + } + + public static formatTestResultsMissing(filePath: string): string { + return `${filePath} holds no test times: the test run wrote none.`; + } + + public static formatTimingRun(run: number): string { + return `Timing run ${run} of ${Resources.timingRuns}...`; + } + + public static formatTimingRunFailed(run: number, exitCode: number | null): string { + return `Timing run ${run} failed with exit code ${exitCode}; the timing check judges only a passing suite.`; + } + public static formatCoverageUnread(exitCode: number | null): string { return `The coverage report exited with code ${exitCode}.`; } diff --git a/src/noldova.tools.pipelines/src/services/coverage-entry-point.ts b/src/noldova.tools.pipelines/src/services/coverage-entry-point.ts index 77c2910..4675f2c 100644 --- a/src/noldova.tools.pipelines/src/services/coverage-entry-point.ts +++ b/src/noldova.tools.pipelines/src/services/coverage-entry-point.ts @@ -6,28 +6,14 @@ * LICENSE file in the root directory of this source tree. */ -import { createRequire } from "node:module"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -import type { ITestingModule } from "../interfaces/i-testing-module.js"; import { Resources } from "../resources.js"; +import { TestingModuleLoader } from "./testing-module-loader.js"; const values = process.argv.slice(2); -let modulePath: string | null = null; -try { - modulePath = createRequire(path.resolve(Resources.manifestFileName)).resolve(Resources.testingPackageName); -} -catch (error: unknown) { - if (!(error instanceof Error && Reflect.get(error, Resources.codeField) === Resources.moduleNotFoundCode)) - throw error; - - process.stderr.write(`${Resources.testingMissing}${Resources.lineSeparator}`); +const testing = await TestingModuleLoader.loadAsync(); +if (testing === null) process.exitCode = Resources.coverageUnreadExitCode; -} - -if (modulePath !== null) { - const testing = await import(pathToFileURL(modulePath).href) as ITestingModule; +else { const projects = (JSON.parse(String(values[2])) as [name: string, packageFolder: string, sourceFolder: string][]) .map(([name, packageFolder, sourceFolder]) => new testing.CoverageProject(name, packageFolder, sourceFolder)); const coverage = await new testing.CoverageAnalyzer().analyzeAsync(String(values[0]), projects); diff --git a/src/noldova.tools.pipelines/src/services/test-step.ts b/src/noldova.tools.pipelines/src/services/test-step.ts index c017130..ee3d857 100644 --- a/src/noldova.tools.pipelines/src/services/test-step.ts +++ b/src/noldova.tools.pipelines/src/services/test-step.ts @@ -23,6 +23,7 @@ import type { TypeScript } from "./type-script.js"; export class TestStep implements IStep { private static readonly COVERAGE_ENTRY_POINT_PATH: string = fileURLToPath(new URL(Resources.coverageEntryPointPath, import.meta.url)); + private static readonly TIMES_ENTRY_POINT_PATH: string = fileURLToPath(new URL(Resources.testTimesEntryPointPath, import.meta.url)); private readonly typeScript: TypeScript; private readonly processRunner: ProcessRunner; @@ -61,6 +62,9 @@ export class TestStep implements IStep { } private async testAsync(config: BuildConfig, options: ReadonlySet, filters: readonly string[]): Promise { + if (config.testTimeoutMilliseconds === undefined) + return StepResult.failed(Resources.testTimeoutMissing); + const projectValues: string[] = []; for (const buildPackage of config.packages) { const projectPath = Resources.formatPath(buildPackage.sourceFolder, Resources.testsProjectPath); @@ -84,7 +88,13 @@ export class TestStep implements IStep { return StepResult.failed(Resources.testsMissing); const coverageFolder = this.resolve(Resources.coverageFolder); - const environment: Record = { [Resources.testFiltersVariable]: JSON.stringify(filters) }; + const resultsFile = this.resolve(Resources.testResultsFile); + fs.rmSync(resultsFile, { force: true }); + const environment: Record = { + [Resources.testFiltersVariable]: JSON.stringify(filters), + [Resources.testTimeoutVariable]: String(config.testTimeoutMilliseconds), + [Resources.testResultsVariable]: resultsFile + }; if (options.has(Resources.skipTestDetailsOption)) environment[Resources.skipTestDetailsVariable] = Resources.enabledValue; if (filters.length === 0) { @@ -95,6 +105,10 @@ export class TestStep implements IStep { const exitCode = await this.processRunner.runVisibleAsync(process.execPath, [Resources.sourceMapsOption, Resources.testEntryPath, ...projectValues], this.typeScript.workingDirectory, environment); const problems = exitCode === Resources.successExitCode ? [] : [Resources.formatTestsFailed(exitCode)]; + const timesExitCode = await this.processRunner.runVisibleAsync(process.execPath, + [TestStep.TIMES_ENTRY_POINT_PATH, Resources.slowestMode, resultsFile, String(Resources.slowestTestCount)], this.typeScript.workingDirectory, {}); + if (timesExitCode !== Resources.successExitCode) + problems.push(Resources.formatTimesUnread(timesExitCode)); if (filters.length > 0) return problems.length === 0 ? StepResult.succeeded(Resources.filteredCoverageSkipped) : StepResult.failed([...problems, Resources.filteredCoverageSkipped].join(Resources.lineSeparator)); diff --git a/src/noldova.tools.pipelines/src/services/test-times-entry-point.ts b/src/noldova.tools.pipelines/src/services/test-times-entry-point.ts new file mode 100644 index 0000000..209a003 --- /dev/null +++ b/src/noldova.tools.pipelines/src/services/test-times-entry-point.ts @@ -0,0 +1,43 @@ +/** + * @license + * Copyright (c) Noldova. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + + +import fs from "node:fs"; + +import { Resources } from "../resources.js"; +import { TestingModuleLoader } from "./testing-module-loader.js"; + +// Reads the test times a run wrote through the checkout's @noldova/context.testing: "slowest " prints the slowest tests, +// and "judge ..." judges each test file's median time over the runs, failing when one is over. +const [mode, ...values] = process.argv.slice(2); +const testing = await TestingModuleLoader.loadAsync(); +if (testing === null) + process.exitCode = Resources.timesUnreadExitCode; +else if (mode === Resources.slowestMode) { + const resultsFile = String(values[0]); + if (!fs.existsSync(resultsFile)) { + process.stderr.write(`${Resources.formatTestResultsMissing(resultsFile)}${Resources.lineSeparator}`); + process.exitCode = Resources.timesUnreadExitCode; + } + else { + for (const line of testing.TestTimeRecord.formatSlowest(testing.TestTimeRecord.parseAll(fs.readFileSync(resultsFile, Resources.textEncoding)), Number(values[1]))) + process.stdout.write(`${line}${Resources.lineSeparator}`); + } +} +else { + const [budget, margin, readme, ...resultsFiles] = values; + const readmePath = String(readme); + const listed = testing.TestTimingCheck.readListed(fs.existsSync(readmePath) ? fs.readFileSync(readmePath, Resources.textEncoding) : ""); + const runs = resultsFiles.map(t => testing.TestTimeRecord.parseAll(fs.readFileSync(t, Resources.textEncoding))); + const timings = new testing.TestTimingCheck(Number(budget), Number(margin), listed).judge(runs); + for (const timing of timings.filter(t => t.isOver || t.isListed)) + process.stdout.write(`${timing.format()}${Resources.lineSeparator}`); + + process.stdout.write(`${testing.TestTimingCheck.formatSummary(timings, runs.length)}${Resources.lineSeparator}`); + process.exitCode = timings.some(t => t.isOver) ? Resources.failureExitCode : Resources.successExitCode; +} diff --git a/src/noldova.tools.pipelines/src/services/testing-module-loader.ts b/src/noldova.tools.pipelines/src/services/testing-module-loader.ts new file mode 100644 index 0000000..f7ec5ba --- /dev/null +++ b/src/noldova.tools.pipelines/src/services/testing-module-loader.ts @@ -0,0 +1,35 @@ +/** + * @license + * Copyright (c) Noldova. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + + +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import type { ITestingModule } from "../interfaces/i-testing-module.js"; +import { Resources } from "../resources.js"; + +// Loads the checkout's own @noldova/context.testing, the version its tests run with, rather than one of the build tool's. +export class TestingModuleLoader { + // Returns the module, or null after writing why to standard error when the checkout doesn't have it installed. + public static async loadAsync(): Promise { + let modulePath: string; + try { + modulePath = createRequire(path.resolve(Resources.manifestFileName)).resolve(Resources.testingPackageName); + } + catch (error: unknown) { + if (!(error instanceof Error && Reflect.get(error, Resources.codeField) === Resources.moduleNotFoundCode)) + throw error; + + process.stderr.write(`${Resources.testingMissing}${Resources.lineSeparator}`); + return null; + } + + return await import(pathToFileURL(modulePath).href) as ITestingModule; + } +} diff --git a/src/noldova.tools.pipelines/src/services/timing-step.ts b/src/noldova.tools.pipelines/src/services/timing-step.ts new file mode 100644 index 0000000..2af7cc7 --- /dev/null +++ b/src/noldova.tools.pipelines/src/services/timing-step.ts @@ -0,0 +1,102 @@ +/** + * @license + * Copyright (c) Noldova. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { PipelineException } from "../exceptions/pipeline.exception.js"; +import type { IStep } from "../interfaces/i-step.js"; +import type { IStepOutput } from "../interfaces/i-step-output.js"; +import { BuildConfig } from "../models/build-config.js"; +import { StepResult } from "../models/step-result.js"; +import { Resources } from "../resources.js"; +import type { ProcessRunner } from "./process-runner.js"; + +export class TimingStep implements IStep { + private static readonly TIMES_ENTRY_POINT_PATH: string = fileURLToPath(new URL(Resources.testTimesEntryPointPath, import.meta.url)); + + private readonly workingDirectory: string; + private readonly processRunner: ProcessRunner; + private readonly output: IStepOutput; + + public readonly name: string = Resources.timingStepName; + public readonly usage: string = Resources.timingStepUsage; + + public constructor(workingDirectory: string, processRunner: ProcessRunner, output: IStepOutput) { + this.workingDirectory = workingDirectory; + this.processRunner = processRunner; + this.output = output; + } + + public async runAsync(values: readonly string[]): Promise { + if (values.length !== 0) + return StepResult.failed(Resources.formatStepValuesInvalid(this.name, this.usage)); + + try { + return await this.timeAsync(BuildConfig.read(this.workingDirectory)); + } + catch (error) { + if (error instanceof PipelineException) + return StepResult.failed(error.message); + + throw error; + } + } + + private async timeAsync(config: BuildConfig): Promise { + if (config.testTimeoutMilliseconds === undefined) + return StepResult.failed(Resources.testTimeoutMissing); + + const tested = config.packages.filter(t => fs.statSync(this.resolve(Resources.formatPath(t.sourceFolder, Resources.testsProjectPath)), { throwIfNoEntry: false })?.isFile() === true); + if (tested.length === 0) + return StepResult.failed(Resources.testsMissing); + if (tested.some(t => fs.statSync(this.resolve(t.testsFolder), { throwIfNoEntry: false })?.isDirectory() !== true)) + return StepResult.failed(Resources.timingNotBuilt); + + const timingFolder = this.resolve(Resources.timingFolder); + fs.rmSync(timingFolder, { recursive: true, force: true }); + fs.mkdirSync(timingFolder, { recursive: true }); + const resultsFiles: string[] = []; + for (let run = 1; run <= Resources.timingRuns; run++) { + const resultsFile = path.join(timingFolder, `run-${run}.json`); + this.output.writeProgressStart(Resources.formatTimingRun(run)); + const exitCode = await this.processRunner.runVisibleAsync(process.execPath, + [Resources.sourceMapsOption, Resources.testEntryPath, ...tested.flatMap(t => [t.name, t.testsFolder])], + this.workingDirectory, + { + [Resources.testFiltersVariable]: JSON.stringify([]), + [Resources.testTimeoutVariable]: String(config.testTimeoutMilliseconds), + [Resources.testResultsVariable]: resultsFile, + [Resources.skipTestDetailsVariable]: Resources.enabledValue + }); + if (exitCode !== Resources.successExitCode) { + this.output.writeProgress(Resources.progressFailed); + return StepResult.failed(Resources.formatTimingRunFailed(run, exitCode)); + } + + this.output.writeProgress(Resources.progressDone); + + resultsFiles.push(resultsFile); + } + + const judged = await this.processRunner.runVisibleAsync(process.execPath, + [TimingStep.TIMES_ENTRY_POINT_PATH, Resources.judgeMode, String(Resources.timingBudgetMilliseconds), String(Resources.timingMargin), this.resolve(Resources.readmeFileName), ...resultsFiles], + this.workingDirectory, + {}); + if (judged === Resources.successExitCode) + return StepResult.succeeded(Resources.timingPassed); + + return StepResult.failed(judged === Resources.failureExitCode ? Resources.timingOver : Resources.formatTimesUnread(judged)); + } + + private resolve(filePath: string): string { + return path.join(this.workingDirectory, filePath); + } +} diff --git a/src/noldova.tools.pipelines/tests/api/index.test.ts b/src/noldova.tools.pipelines/tests/api/index.test.ts index 15df191..cc40fbf 100644 --- a/src/noldova.tools.pipelines/tests/api/index.test.ts +++ b/src/noldova.tools.pipelines/tests/api/index.test.ts @@ -13,6 +13,6 @@ import * as api from "@noldova/tools.pipelines"; export class PipelinesApiTests { @TestMethod public exportsTheCompleteCatalog(): void { - Assert.areEqual("ArchiveCheck,ArchiveStep,BuildConfig,BuildException,BuildPackage,BuildStep,CommandLine,DependentsStep,Git,InstalledStep,LineEndingsStep,NightlyStep,Npm,PackageArchive,PinsStep,PipelineException,ProcessException,ProcessResult,ProcessRunner,PruneStep,PublishStep,RelockStep,RelockedFiles,ReproducibleStep,SourceStep,StepResult,TagStep,Tar,TestStep,ToolchainStep,TrackedFile,TypeScript,UnchangedStep,Version,VersionException", Object.keys(api).sort().join(",")); + Assert.areEqual("ArchiveCheck,ArchiveStep,BuildConfig,BuildException,BuildPackage,BuildStep,CommandLine,DependentsStep,Git,InstalledStep,LineEndingsStep,NightlyStep,Npm,PackageArchive,PinsStep,PipelineException,ProcessException,ProcessResult,ProcessRunner,PruneStep,PublishStep,RelockStep,RelockedFiles,ReproducibleStep,SourceStep,StepResult,TagStep,Tar,TestStep,TimingStep,ToolchainStep,TrackedFile,TypeScript,UnchangedStep,Version,VersionException", Object.keys(api).sort().join(",")); } } diff --git a/src/noldova.tools.pipelines/tests/models/build-config.test.ts b/src/noldova.tools.pipelines/tests/models/build-config.test.ts index 01e886c..3ceeb68 100644 --- a/src/noldova.tools.pipelines/tests/models/build-config.test.ts +++ b/src/noldova.tools.pipelines/tests/models/build-config.test.ts @@ -51,6 +51,27 @@ export class BuildConfigTests { Assert.areEqual(buildPackage, config.packages[0]); } + @TestMethod + public readsTheSuitesHangLimit(): void { + Assert.areEqual(45000, BuildConfig.parse("{\"testTimeoutMilliseconds\":45000,\"packages\":[{\"name\":\"noldova.example\"}]}").testTimeoutMilliseconds); + Assert.isUndefined(BuildConfig.parse("{\"packages\":[{\"name\":\"noldova.example\"}]}").testTimeoutMilliseconds); + Assert.areEqual(30000, new BuildConfig("@noldova", [new BuildPackage("noldova.example")], 30000).testTimeoutMilliseconds); + } + + @TestData(0) + @TestData(-1) + @TestMethod + public refusesAHangLimitThatIsNotPositive(milliseconds: number): void { + const exception = Assert.throws(() => BuildConfig.parse(`{"testTimeoutMilliseconds":${milliseconds},"packages":[{"name":"noldova.example"}]}`), BuildException); + + Assert.areEqual(`build.json is invalid: testTimeoutMilliseconds is ${milliseconds}, not a positive number of milliseconds.`, exception.message); + } + + @TestMethod + public refusesAHangLimitThatIsNotAnInteger(): void { + Assert.throws(() => BuildConfig.parse("{\"testTimeoutMilliseconds\":1.5,\"packages\":[{\"name\":\"noldova.example\"}]}"), JsonException); + } + @TestMethod public readsTheBuildFileOfACheckout(): void { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tools-pipelines-")); diff --git a/src/noldova.tools.pipelines/tests/services/test-step.test.ts b/src/noldova.tools.pipelines/tests/services/test-step.test.ts index e5bf5e2..a76a347 100644 --- a/src/noldova.tools.pipelines/tests/services/test-step.test.ts +++ b/src/noldova.tools.pipelines/tests/services/test-step.test.ts @@ -66,7 +66,7 @@ class Checkout implements Disposable { public constructor(files: Readonly> = {}) { this.write({ - "build.json": "{\"packages\":[{\"name\":\"noldova.a\"},{\"name\":\"noldova.b\",\"path\":\"group/noldova.b\"}]}", + "build.json": "{\"testTimeoutMilliseconds\":30000,\"packages\":[{\"name\":\"noldova.a\"},{\"name\":\"noldova.b\",\"path\":\"group/noldova.b\"}]}", "src/noldova.a/tests/tsconfig.json": "{}", ...files }); @@ -91,6 +91,7 @@ class Checkout implements Disposable { @TestClass export class TestStepTests { private static readonly COVERAGE_ENTRY_POINT_PATH: string = fileURLToPath(import.meta.resolve("@noldova/tools.pipelines/services/coverage-entry-point.js")); + private static readonly TIMES_ENTRY_POINT_PATH: string = fileURLToPath(import.meta.resolve("@noldova/tools.pipelines/services/test-times-entry-point.js")); @TestMethod public namesItselfAndItsValues(): void { @@ -120,11 +121,17 @@ export class TestStepTests { ["noldova.a", path.join(checkout.directory, "node_modules/@noldova/a"), path.join(checkout.directory, "src/noldova.a/src")], ["noldova.b", path.join(checkout.directory, "node_modules/@noldova/b"), path.join(checkout.directory, "src/group/noldova.b/src")] ]; + const resultsFile = path.join(checkout.directory, "_build/test-results.json"); Assert.areEqual(JSON.stringify([ [process.execPath, "--enable-source-maps", "node_modules/@noldova/context.testing/services/execution/test-run-entry-point.js", "noldova.a", "_build/tests/noldova.a", checkout.directory], + [process.execPath, TestStepTests.TIMES_ENTRY_POINT_PATH, "slowest", resultsFile, "10", checkout.directory], [process.execPath, TestStepTests.COVERAGE_ENTRY_POINT_PATH, coverageFolder, "", JSON.stringify(projects), checkout.directory] ]), JSON.stringify(runner.calls)); - Assert.areEqual(JSON.stringify([{ CONTEXT_TEST_FILTERS: "[]", NODE_V8_COVERAGE: coverageFolder }, {}]), JSON.stringify(runner.environments)); + Assert.areEqual(JSON.stringify([ + { CONTEXT_TEST_FILTERS: "[]", CONTEXT_TEST_TIMEOUT_MILLISECONDS: "30000", CONTEXT_TEST_RESULTS_FILE: resultsFile, NODE_V8_COVERAGE: coverageFolder }, + {}, + {} + ]), JSON.stringify(runner.environments)); Assert.areEqual("progress start: Compiling the tests of noldova.a...|progress: Done.", lines.join("|")); } @@ -136,7 +143,7 @@ export class TestStepTests { await new TestStep(new FakeTypeScript(checkout.directory), runner, new RecordingOutput([])).runAsync(["--skip-test-details", "--skip-coverage-details"]); Assert.areEqual("1", runner.environments[0]?.["CONTEXT_SKIP_TEST_DETAILS"]); - Assert.areEqual("1", runner.calls[1]?.[3]); + Assert.areEqual("1", runner.calls[2]?.[3]); } @TestData(0, "The coverage gate is skipped for a filtered run.") @@ -150,17 +157,23 @@ export class TestStepTests { Assert.areEqual(exitCode === 0, result.isSuccessful); Assert.areEqual(message, result.message); - Assert.areEqual(JSON.stringify([{ CONTEXT_TEST_FILTERS: "[\"ArchiveStep\",\"noldova.a\"]" }]), JSON.stringify(runner.environments)); + const resultsFile = path.join(checkout.directory, "_build/test-results.json"); + Assert.areEqual( + JSON.stringify([{ CONTEXT_TEST_FILTERS: "[\"ArchiveStep\",\"noldova.a\"]", CONTEXT_TEST_TIMEOUT_MILLISECONDS: "30000", CONTEXT_TEST_RESULTS_FILE: resultsFile }, {}]), + JSON.stringify(runner.environments)); + Assert.areEqual(JSON.stringify([process.execPath, TestStepTests.TIMES_ENTRY_POINT_PATH, "slowest", resultsFile, "10", checkout.directory]), JSON.stringify(runner.calls[1])); } - @TestData(null, 1, "The tests failed with exit code null.\nCoverage is incomplete.") - @TestData(0, 2, "The coverage report exited with code 2.") - @TestData(0, null, "The coverage report exited with code null.") + @TestData(null, 0, 1, "The tests failed with exit code null.\nCoverage is incomplete.") + @TestData(0, 0, 2, "The coverage report exited with code 2.") + @TestData(0, 0, null, "The coverage report exited with code null.") + @TestData(1, 2, 0, "The tests failed with exit code 1.\nThe test times were not read: their report exited with code 2.") + @TestData(0, null, 1, "The test times were not read: their report exited with code null.\nCoverage is incomplete.") @TestMethod - public async failsForFailedTestsOrCoverageAsync(testsExitCode: number | null, coverageExitCode: number | null, message: string): Promise { + public async failsForFailedTestsTimesOrCoverageAsync(testsExitCode: number | null, timesExitCode: number | null, coverageExitCode: number | null, message: string): Promise { using checkout = new Checkout(); - const result = await new TestStep(new FakeTypeScript(checkout.directory), new FakeRunner(testsExitCode, coverageExitCode), new RecordingOutput([])).runAsync([]); + const result = await new TestStep(new FakeTypeScript(checkout.directory), new FakeRunner(testsExitCode, timesExitCode, coverageExitCode), new RecordingOutput([])).runAsync([]); Assert.isFalse(result.isSuccessful); Assert.areEqual(message, result.message); @@ -184,6 +197,31 @@ export class TestStepTests { Assert.areEqual("The checkout has no build.json, which names the packages to build.", result.message); } + @TestMethod + public async failsWithoutTheSuitesHangLimitBeforeCompilingAsync(): Promise { + using checkout = new Checkout({ "build.json": "{\"packages\":[{\"name\":\"noldova.a\"}]}" }); + const typeScript = new FakeTypeScript(checkout.directory); + const runner = new FakeRunner(); + + const result = await new TestStep(typeScript, runner, new RecordingOutput([])).runAsync([]); + + Assert.isFalse(result.isSuccessful); + Assert.areEqual( + "build.json sets no testTimeoutMilliseconds, the suite's hang limit: at least three times its slowest test on any target, with the measurement in README's Checks.", + result.message); + Assert.areEqual(0, typeScript.calls.length); + Assert.areEqual(0, runner.calls.length); + } + + @TestMethod + public async removesAnEarlierRunsTestTimesAsync(): Promise { + using checkout = new Checkout({ "_build/test-results.json": "[]" }); + + await new TestStep(new FakeTypeScript(checkout.directory), new FakeRunner(), new RecordingOutput([])).runAsync([]); + + Assert.isFalse(checkout.exists("_build/test-results.json")); + } + @TestMethod public async failsWhenNoPackageHasTestsAsync(): Promise { using checkout = new Checkout(); diff --git a/src/noldova.tools.pipelines/tests/services/timing-step.test.ts b/src/noldova.tools.pipelines/tests/services/timing-step.test.ts new file mode 100644 index 0000000..4c5bf84 --- /dev/null +++ b/src/noldova.tools.pipelines/tests/services/timing-step.test.ts @@ -0,0 +1,261 @@ +/** + * @license + * Copyright (c) Noldova. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { JsonException } from "@noldova/context.json"; +import { Assert, TestClass, TestData, TestMethod } from "@noldova/context.testing"; +import { ProcessRunner, TimingStep } from "@noldova/tools.pipelines"; + +import { RecordingOutput } from "./fake-update-host.js"; + +class FakeRunner extends ProcessRunner { + public readonly calls: string[][] = []; + public readonly environments: Readonly>[] = []; + private readonly exitCodes: (number | null)[]; + + public constructor(...exitCodes: (number | null)[]) { + super(); + + this.exitCodes = exitCodes; + } + + public override async runVisibleAsync(program: string, values: readonly string[], workingDirectory: string, environment: Readonly>): Promise { + this.calls.push([program, ...values, workingDirectory]); + this.environments.push(environment); + + const [exitCode = 0] = this.exitCodes.splice(0, 1); + + return exitCode; + } +} + +class Checkout implements Disposable { + public readonly directory: string = fs.mkdtempSync(path.join(os.tmpdir(), "tools-pipelines-")); + + public constructor(files: Readonly> = {}) { + this.write({ + "build.json": "{\"testTimeoutMilliseconds\":30000,\"packages\":[{\"name\":\"noldova.a\"},{\"name\":\"noldova.b\"}]}", + "src/noldova.a/tests/tsconfig.json": "{}", + "_build/tests/noldova.a/a.test.js": "", + ...files + }); + } + + public write(files: Readonly>): void { + for (const [filePath, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(this.directory, filePath)), { recursive: true }); + fs.writeFileSync(path.join(this.directory, filePath), content); + } + } + + public exists(filePath: string): boolean { + return fs.existsSync(path.join(this.directory, filePath)); + } + + public [Symbol.dispose](): void { + fs.rmSync(this.directory, { recursive: true, force: true }); + } +} + +@TestClass +export class TimingStepTests { + private static readonly TIMES_ENTRY_POINT_PATH: string = fileURLToPath(import.meta.resolve("@noldova/tools.pipelines/services/test-times-entry-point.js")); + // A stand-in for the checkout's @noldova/context.testing: records are { file, ms }, and a file is over when one of its records exceeds the budget + // with its margin. + private static readonly TESTING_PACKAGE: Readonly> = { + "node_modules/@noldova/context.testing/package.json": "{\"name\":\"@noldova/context.testing\",\"type\":\"module\",\"main\":\"index.js\"}", + "node_modules/@noldova/context.testing/index.js": [ + "export class TestTimeRecord {", + " static parseAll(text) { return JSON.parse(text); }", + " static formatSlowest(records, count) { return [`slowest ${count}`, ...records.map(t => `${t.file} ${t.ms}`)]; }", + "}", + "export class TestTimingCheck {", + " constructor(budget, margin, listed) { this.limit = budget * (1 + margin); this.listed = listed; }", + " static readListed(markdown) { return new Map(markdown.split('\\n').filter(t => t !== '').map(t => [t, 1])); }", + " static formatSummary(timings, runs) { return `summary ${timings.length} files ${runs} runs`; }", + " judge(runs) {", + " const records = runs.flat();", + " return [...new Set([...records.map(t => t.file), ...this.listed.keys()])].map(file => ({", + " isOver: records.some(t => t.file === file && t.ms > this.limit), isListed: this.listed.has(file), format: () => `timing ${file}`", + " }));", + " }", + "}" + ].join("\n") + }; + + @TestMethod + public namesItselfAndItsValues(): void { + const step = new TimingStep(".", new FakeRunner(), new RecordingOutput([])); + + Assert.areEqual("timing", step.name); + Assert.areEqual("", step.usage); + } + + @TestMethod + public async runsTheCompiledTestsThreeTimesAndJudgesTheirTimesAsync(): Promise { + using checkout = new Checkout({ "_build/timing/stale.json": "" }); + const runner = new FakeRunner(); + const lines: string[] = []; + + const result = await new TimingStep(checkout.directory, runner, new RecordingOutput(lines)).runAsync([]); + + Assert.isTrue(result.isSuccessful); + Assert.areEqual("Every test file is within its time.", result.message); + Assert.isFalse(checkout.exists("_build/timing/stale.json")); + const runFiles = [1, 2, 3].map(t => path.join(checkout.directory, "_build/timing", `run-${t}.json`)); + const testRun = [process.execPath, "--enable-source-maps", "node_modules/@noldova/context.testing/services/execution/test-run-entry-point.js", "noldova.a", "_build/tests/noldova.a", checkout.directory]; + Assert.areEqual(JSON.stringify([ + testRun, + testRun, + testRun, + [process.execPath, TimingStepTests.TIMES_ENTRY_POINT_PATH, "judge", "2000", "0.5", path.join(checkout.directory, "README.md"), ...runFiles, checkout.directory] + ]), JSON.stringify(runner.calls)); + Assert.areEqual(JSON.stringify([ + ...runFiles.map(t => ({ CONTEXT_TEST_FILTERS: "[]", CONTEXT_TEST_TIMEOUT_MILLISECONDS: "30000", CONTEXT_TEST_RESULTS_FILE: t, CONTEXT_SKIP_TEST_DETAILS: "1" })), + {} + ]), JSON.stringify(runner.environments)); + Assert.areEqual( + "progress start: Timing run 1 of 3...|progress: Done.|progress start: Timing run 2 of 3...|progress: Done.|progress start: Timing run 3 of 3...|progress: Done.", + lines.join("|")); + } + + @TestMethod + public async stopsAtARunThatFailsAsync(): Promise { + using checkout = new Checkout(); + const runner = new FakeRunner(0, 3); + const lines: string[] = []; + + const result = await new TimingStep(checkout.directory, runner, new RecordingOutput(lines)).runAsync([]); + + Assert.isFalse(result.isSuccessful); + Assert.areEqual("Timing run 2 failed with exit code 3; the timing check judges only a passing suite.", result.message); + Assert.areEqual(2, runner.calls.length); + Assert.areEqual("progress start: Timing run 1 of 3...|progress: Done.|progress start: Timing run 2 of 3...|progress: Failed.", lines.join("|")); + } + + @TestData(1, "A test file is over its time.") + @TestData(2, "The test times were not read: their report exited with code 2.") + @TestData(null, "The test times were not read: their report exited with code null.") + @TestMethod + public async failsForAFileOverItsTimeOrTimesNotReadAsync(exitCode: number | null, message: string): Promise { + using checkout = new Checkout(); + + const result = await new TimingStep(checkout.directory, new FakeRunner(0, 0, 0, exitCode), new RecordingOutput([])).runAsync([]); + + Assert.isFalse(result.isSuccessful); + Assert.areEqual(message, result.message); + } + + @TestMethod + public async failsForAValueAsync(): Promise { + const result = await new TimingStep(".", new FakeRunner(), new RecordingOutput([])).runAsync(["--runs"]); + + Assert.isFalse(result.isSuccessful); + Assert.areEqual("The timing step takes no values.", result.message); + } + + @TestMethod + public async failsWithoutABuildFileAsync(): Promise { + using checkout = new Checkout(); + fs.rmSync(path.join(checkout.directory, "build.json")); + + const result = await new TimingStep(checkout.directory, new FakeRunner(), new RecordingOutput([])).runAsync([]); + + Assert.areEqual("The checkout has no build.json, which names the packages to build.", result.message); + } + + @TestMethod + public async rejectsAFaultAsync(): Promise { + using checkout = new Checkout({ "build.json": "{" }); + + await Assert.throwsAsync(() => new TimingStep(checkout.directory, new FakeRunner(), new RecordingOutput([])).runAsync([]), JsonException); + } + + @TestMethod + public async failsWithoutTheSuitesHangLimitAsync(): Promise { + using checkout = new Checkout({ "build.json": "{\"packages\":[{\"name\":\"noldova.a\"}]}" }); + const runner = new FakeRunner(); + + const result = await new TimingStep(checkout.directory, runner, new RecordingOutput([])).runAsync([]); + + Assert.isFalse(result.isSuccessful); + Assert.areEqual( + "build.json sets no testTimeoutMilliseconds, the suite's hang limit: at least three times its slowest test on any target, with the measurement in README's Checks.", + result.message); + Assert.areEqual(0, runner.calls.length); + } + + @TestMethod + public async failsWhenNoPackageHasTestsAsync(): Promise { + using checkout = new Checkout(); + fs.rmSync(path.join(checkout.directory, "src/noldova.a/tests/tsconfig.json")); + + const result = await new TimingStep(checkout.directory, new FakeRunner(), new RecordingOutput([])).runAsync([]); + + Assert.areEqual("No package has tests.", result.message); + } + + @TestMethod + public async failsWhenTheTestsAreNotCompiledAsync(): Promise { + using checkout = new Checkout(); + fs.rmSync(path.join(checkout.directory, "_build/tests/noldova.a"), { recursive: true }); + const runner = new FakeRunner(); + + const result = await new TimingStep(checkout.directory, runner, new RecordingOutput([])).runAsync([]); + + Assert.areEqual("The tests are not compiled; run the test step first.", result.message); + Assert.areEqual(0, runner.calls.length); + } + + @TestMethod + public async printsTheSlowestTestsOfARunAsync(): Promise { + using checkout = new Checkout({ ...TimingStepTests.TESTING_PACKAGE, "_build/test-results.json": "[{\"file\":\"a\",\"ms\":5}]" }); + + const result = await new ProcessRunner().runAsync(process.execPath, [TimingStepTests.TIMES_ENTRY_POINT_PATH, "slowest", "_build/test-results.json", "10"], checkout.directory); + + Assert.areEqual(0, result.exitCode); + Assert.areEqual("slowest 10\na 5\n", result.output); + } + + @TestMethod + public async failsToPrintTheSlowestTestsWithoutTheRunsTimesAsync(): Promise { + using checkout = new Checkout(TimingStepTests.TESTING_PACKAGE); + + const result = await new ProcessRunner().runAsync(process.execPath, [TimingStepTests.TIMES_ENTRY_POINT_PATH, "slowest", "_build/test-results.json", "10"], checkout.directory); + + Assert.areEqual(2, result.exitCode); + Assert.areEqual("_build/test-results.json holds no test times: the test run wrote none.\n", result.errorOutput); + } + + @TestData("[{\"file\":\"a\",\"ms\":200}]", "", 1, "timing a\nsummary 1 files 2 runs\n") + @TestData("[{\"file\":\"a\",\"ms\":100}]", "", 0, "summary 1 files 2 runs\n") + @TestData("[{\"file\":\"a\",\"ms\":100}]", "a\n", 0, "timing a\nsummary 1 files 2 runs\n") + @TestMethod + public async judgesTheRunsTimesWithTheReadmesListAsync(secondRun: string, readme: string, exitCode: number, output: string): Promise { + using checkout = new Checkout({ ...TimingStepTests.TESTING_PACKAGE, "run-1.json": "[{\"file\":\"a\",\"ms\":100}]", "run-2.json": secondRun, ...readme === "" ? {} : { "README.md": readme } }); + + const result = await new ProcessRunner().runAsync(process.execPath, [TimingStepTests.TIMES_ENTRY_POINT_PATH, "judge", "100", "0.5", "README.md", "run-1.json", "run-2.json"], checkout.directory); + + Assert.areEqual(exitCode, result.exitCode); + Assert.areEqual(output, result.output); + } + + @TestMethod + public async failsToJudgeWithoutTheTestingPackageAsync(): Promise { + using checkout = new Checkout(); + + const result = await new ProcessRunner().runAsync(process.execPath, [TimingStepTests.TIMES_ENTRY_POINT_PATH, "judge", "100", "0.5", "README.md"], checkout.directory); + + Assert.areEqual(2, result.exitCode); + Assert.areEqual("@noldova/context.testing is not installed; the repository's devDependencies name it.\n", result.errorOutput); + } +} From fd43f2be621902a586820e2dc49ace3a3b5d5bf5 Mon Sep 17 00:00:00 2001 From: Rostislav Rotaru Date: Sat, 14 Nov 2026 15:38:15 -0800 Subject: [PATCH 2/4] Tell unread test times from a file over its time The times report exits with its own code, 3, for a file over its time, and with the unread code, 2, for a results file that is missing or isn't test times, in both modes; the timing step names each. A fault still ends with its stack trace. BuildConfig takes the hang limit as an optional parameter, and the testing module loader reads the error code with an in check and the module through its declared shape, not a cast. Agent: DEV1 (developer) Agent-Model: Anthropic Opus 5.5, Medium --- src/noldova.tools.pipelines/README.md | 1 + .../src/api/index.d.ts | 4 +- .../src/interfaces/i-testing-module.ts | 1 + .../src/models/build-config.ts | 4 +- src/noldova.tools.pipelines/src/resources.ts | 7 ++- .../src/services/test-times-entry-point.ts | 51 ++++++++++++++----- .../src/services/testing-module-loader.ts | 6 ++- .../src/services/timing-step.ts | 2 +- .../tests/services/timing-step.test.ts | 47 +++++++++++++++-- 9 files changed, 99 insertions(+), 24 deletions(-) diff --git a/src/noldova.tools.pipelines/README.md b/src/noldova.tools.pipelines/README.md index d73bc74..b2c2070 100644 --- a/src/noldova.tools.pipelines/README.md +++ b/src/noldova.tools.pipelines/README.md @@ -430,6 +430,7 @@ It writes `Compiling the tests of ...` for each, and ends the line with Run it after the test step, which compiles the tests: it runs them three times without coverage, writes `Timing run of 3...` for each, and judges each test file's time, the sum of its tests' durations, through the repository's `@noldova/context.testing`. It fails a file whose median over the runs is over 2 seconds plus a margin of 50%, since a busy machine slows every file and the median of three runs drops one slow run but not a slower machine. A file slower than that budget is listed in the repository's README, after the line `Test files slower than the timing check's budget:`, as a table row such as ``| `noldova.example/services/a.test.js` | 6900 ms |``; its median may exceed its own time by the same margin, and a listed file within the budget is named as stale, failing nothing. +A file over its time fails the step with `A test file is over its time.`; a run's times file that is missing or isn't test times, as from a `@noldova/context.testing` before 0.0.8, which writes none, fails it with `The test times were not read: …`, after the report has named the file. ## License diff --git a/src/noldova.tools.pipelines/src/api/index.d.ts b/src/noldova.tools.pipelines/src/api/index.d.ts index 9b8a34b..01be51c 100644 --- a/src/noldova.tools.pipelines/src/api/index.d.ts +++ b/src/noldova.tools.pipelines/src/api/index.d.ts @@ -606,7 +606,7 @@ export declare class BuildConfig { * * @param scope The npm scope the packages publish under. * @param packages The packages, in build order. - * @param testTimeoutMilliseconds The suite's hang limit in milliseconds, or undefined for none. + * @param testTimeoutMilliseconds The suite's hang limit in milliseconds, left out for none. * @example * ```ts * import { BuildConfig, BuildPackage } from "@noldova/tools.pipelines"; @@ -614,7 +614,7 @@ export declare class BuildConfig { * const config = new BuildConfig("@noldova", [new BuildPackage("noldova.tools.pipelines")]); * ``` */ - public constructor(scope: string, packages: readonly BuildPackage[], testTimeoutMilliseconds?: number | undefined); + public constructor(scope: string, packages: readonly BuildPackage[], testTimeoutMilliseconds?: number); /** * The packages' npm names, such as `@noldova/tools.pipelines`. diff --git a/src/noldova.tools.pipelines/src/interfaces/i-testing-module.ts b/src/noldova.tools.pipelines/src/interfaces/i-testing-module.ts index be77946..5e34205 100644 --- a/src/noldova.tools.pipelines/src/interfaces/i-testing-module.ts +++ b/src/noldova.tools.pipelines/src/interfaces/i-testing-module.ts @@ -17,6 +17,7 @@ export interface ITestFileTiming { } export interface ITestingModule { + readonly TestingException: abstract new (...values: never[]) => Error; readonly CoverageProject: new (name: string, packageFolder: string, sourceFolder: string) => unknown; readonly CoverageAnalyzer: new () => { analyzeAsync(coverageFolder: string, projects: readonly unknown[]): Promise }; readonly CoverageReportWriter: new () => { formatLines(coverage: ICoverage, skipDetails: boolean): readonly string[] }; diff --git a/src/noldova.tools.pipelines/src/models/build-config.ts b/src/noldova.tools.pipelines/src/models/build-config.ts index b5fddd6..9541059 100644 --- a/src/noldova.tools.pipelines/src/models/build-config.ts +++ b/src/noldova.tools.pipelines/src/models/build-config.ts @@ -20,7 +20,7 @@ export class BuildConfig { public readonly packages: readonly BuildPackage[]; public readonly testTimeoutMilliseconds: number | undefined; - public constructor(scope: string, packages: readonly BuildPackage[], testTimeoutMilliseconds: number | undefined = undefined) { + public constructor(scope: string, packages: readonly BuildPackage[], testTimeoutMilliseconds?: number) { this.scope = scope; this.packages = packages; this.testTimeoutMilliseconds = testTimeoutMilliseconds; @@ -61,7 +61,7 @@ export class BuildConfig { if (testTimeoutMilliseconds !== undefined && testTimeoutMilliseconds <= 0) throw new BuildException(Resources.formatBuildConfigInvalid(Resources.formatTestTimeoutInvalid(testTimeoutMilliseconds))); - return new BuildConfig(scope, packages, testTimeoutMilliseconds); + return testTimeoutMilliseconds === undefined ? new BuildConfig(scope, packages) : new BuildConfig(scope, packages, testTimeoutMilliseconds); } private static readPackage(entry: JsonReader, scope: string): BuildPackage { diff --git a/src/noldova.tools.pipelines/src/resources.ts b/src/noldova.tools.pipelines/src/resources.ts index fab2f22..7e292bc 100644 --- a/src/noldova.tools.pipelines/src/resources.ts +++ b/src/noldova.tools.pipelines/src/resources.ts @@ -131,7 +131,7 @@ export class Resources { public static readonly npmJsonValues: readonly string[] = ["--json"]; public static readonly npmLockValues: readonly string[] = ["install", "--package-lock-only", "--ignore-scripts", "--no-audit", "--no-fund"]; public static readonly npmNotFoundCode: string = "E404"; - public static readonly codeField: string = "code"; + public static readonly codeField: "code" = "code"; public static readonly moduleNotFoundCode: string = "MODULE_NOT_FOUND"; public static readonly npmPreferOnlineValues: readonly string[] = ["--prefer-online"]; public static readonly npmProgram: string = "npm"; @@ -273,6 +273,7 @@ export class Resources { public static readonly slowestMode: string = "slowest"; public static readonly judgeMode: string = "judge"; public static readonly slowestTestCount: number = 10; + public static readonly timesOverExitCode: number = 3; public static readonly timesUnreadExitCode: number = 2; public static readonly timingStepName: string = "timing"; public static readonly timingStepUsage: string = ""; @@ -1120,6 +1121,10 @@ export class Resources { return `${filePath} holds no test times: the test run wrote none.`; } + public static formatTestResultsInvalid(filePath: string, reason: string): string { + return `${filePath} holds no test times: ${reason}`; + } + public static formatTimingRun(run: number): string { return `Timing run ${run} of ${Resources.timingRuns}...`; } diff --git a/src/noldova.tools.pipelines/src/services/test-times-entry-point.ts b/src/noldova.tools.pipelines/src/services/test-times-entry-point.ts index 209a003..c246aeb 100644 --- a/src/noldova.tools.pipelines/src/services/test-times-entry-point.ts +++ b/src/noldova.tools.pipelines/src/services/test-times-entry-point.ts @@ -9,23 +9,22 @@ import fs from "node:fs"; +import type { ITestingModule } from "../interfaces/i-testing-module.js"; import { Resources } from "../resources.js"; import { TestingModuleLoader } from "./testing-module-loader.js"; // Reads the test times a run wrote through the checkout's @noldova/context.testing: "slowest " prints the slowest tests, -// and "judge ..." judges each test file's median time over the runs, failing when one is over. +// and "judge ..." judges each test file's median time over the runs, exiting with timesOverExitCode when one is over and timesUnreadExitCode when a results file is missing or isn't test times. const [mode, ...values] = process.argv.slice(2); const testing = await TestingModuleLoader.loadAsync(); if (testing === null) process.exitCode = Resources.timesUnreadExitCode; else if (mode === Resources.slowestMode) { - const resultsFile = String(values[0]); - if (!fs.existsSync(resultsFile)) { - process.stderr.write(`${Resources.formatTestResultsMissing(resultsFile)}${Resources.lineSeparator}`); + const runs = readRuns(testing, [String(values[0])]); + if (runs === null) process.exitCode = Resources.timesUnreadExitCode; - } else { - for (const line of testing.TestTimeRecord.formatSlowest(testing.TestTimeRecord.parseAll(fs.readFileSync(resultsFile, Resources.textEncoding)), Number(values[1]))) + for (const line of testing.TestTimeRecord.formatSlowest(runs.flat(), Number(values[1]))) process.stdout.write(`${line}${Resources.lineSeparator}`); } } @@ -33,11 +32,39 @@ else { const [budget, margin, readme, ...resultsFiles] = values; const readmePath = String(readme); const listed = testing.TestTimingCheck.readListed(fs.existsSync(readmePath) ? fs.readFileSync(readmePath, Resources.textEncoding) : ""); - const runs = resultsFiles.map(t => testing.TestTimeRecord.parseAll(fs.readFileSync(t, Resources.textEncoding))); - const timings = new testing.TestTimingCheck(Number(budget), Number(margin), listed).judge(runs); - for (const timing of timings.filter(t => t.isOver || t.isListed)) - process.stdout.write(`${timing.format()}${Resources.lineSeparator}`); + const runs = readRuns(testing, resultsFiles); + if (runs === null) + process.exitCode = Resources.timesUnreadExitCode; + else { + const timings = new testing.TestTimingCheck(Number(budget), Number(margin), listed).judge(runs); + for (const timing of timings.filter(t => t.isOver || t.isListed)) + process.stdout.write(`${timing.format()}${Resources.lineSeparator}`); + + process.stdout.write(`${testing.TestTimingCheck.formatSummary(timings, runs.length)}${Resources.lineSeparator}`); + process.exitCode = timings.some(t => t.isOver) ? Resources.timesOverExitCode : Resources.successExitCode; + } +} + +// Each results file's records, or null after writing to standard error why one of them is missing or isn't test times. +function readRuns(module: ITestingModule, resultsFiles: readonly string[]): (readonly unknown[])[] | null { + const runs: (readonly unknown[])[] = []; + for (const resultsFile of resultsFiles) { + if (!fs.existsSync(resultsFile)) { + process.stderr.write(`${Resources.formatTestResultsMissing(resultsFile)}${Resources.lineSeparator}`); + return null; + } + + try { + runs.push(module.TestTimeRecord.parseAll(fs.readFileSync(resultsFile, Resources.textEncoding))); + } + catch (error: unknown) { + if (!(error instanceof module.TestingException)) + throw error; + + process.stderr.write(`${Resources.formatTestResultsInvalid(resultsFile, error.message)}${Resources.lineSeparator}`); + return null; + } + } - process.stdout.write(`${testing.TestTimingCheck.formatSummary(timings, runs.length)}${Resources.lineSeparator}`); - process.exitCode = timings.some(t => t.isOver) ? Resources.failureExitCode : Resources.successExitCode; + return runs; } diff --git a/src/noldova.tools.pipelines/src/services/testing-module-loader.ts b/src/noldova.tools.pipelines/src/services/testing-module-loader.ts index f7ec5ba..bc376c4 100644 --- a/src/noldova.tools.pipelines/src/services/testing-module-loader.ts +++ b/src/noldova.tools.pipelines/src/services/testing-module-loader.ts @@ -23,13 +23,15 @@ export class TestingModuleLoader { modulePath = createRequire(path.resolve(Resources.manifestFileName)).resolve(Resources.testingPackageName); } catch (error: unknown) { - if (!(error instanceof Error && Reflect.get(error, Resources.codeField) === Resources.moduleNotFoundCode)) + if (!(error instanceof Error && Resources.codeField in error && error.code === Resources.moduleNotFoundCode)) throw error; process.stderr.write(`${Resources.testingMissing}${Resources.lineSeparator}`); return null; } - return await import(pathToFileURL(modulePath).href) as ITestingModule; + // The checkout's module is found by path at run time, so no static import can name it; the interface is the shape the steps read. + const testing: ITestingModule = await import(pathToFileURL(modulePath).href); + return testing; } } diff --git a/src/noldova.tools.pipelines/src/services/timing-step.ts b/src/noldova.tools.pipelines/src/services/timing-step.ts index 2af7cc7..617a2f6 100644 --- a/src/noldova.tools.pipelines/src/services/timing-step.ts +++ b/src/noldova.tools.pipelines/src/services/timing-step.ts @@ -93,7 +93,7 @@ export class TimingStep implements IStep { if (judged === Resources.successExitCode) return StepResult.succeeded(Resources.timingPassed); - return StepResult.failed(judged === Resources.failureExitCode ? Resources.timingOver : Resources.formatTimesUnread(judged)); + return StepResult.failed(judged === Resources.timesOverExitCode ? Resources.timingOver : Resources.formatTimesUnread(judged)); } private resolve(filePath: string): string { diff --git a/src/noldova.tools.pipelines/tests/services/timing-step.test.ts b/src/noldova.tools.pipelines/tests/services/timing-step.test.ts index 4c5bf84..ab4045a 100644 --- a/src/noldova.tools.pipelines/tests/services/timing-step.test.ts +++ b/src/noldova.tools.pipelines/tests/services/timing-step.test.ts @@ -74,8 +74,13 @@ export class TimingStepTests { private static readonly TESTING_PACKAGE: Readonly> = { "node_modules/@noldova/context.testing/package.json": "{\"name\":\"@noldova/context.testing\",\"type\":\"module\",\"main\":\"index.js\"}", "node_modules/@noldova/context.testing/index.js": [ + "export class TestingException extends Error {}", "export class TestTimeRecord {", - " static parseAll(text) { return JSON.parse(text); }", + " static parseAll(text) {", + " if (text === 'fault') throw new TypeError('A fault.');", + " try { return JSON.parse(text); }", + " catch { throw new TestingException('The file is not JSON.'); }", + " }", " static formatSlowest(records, count) { return [`slowest ${count}`, ...records.map(t => `${t.file} ${t.ms}`)]; }", "}", "export class TestTimingCheck {", @@ -142,8 +147,9 @@ export class TimingStepTests { Assert.areEqual("progress start: Timing run 1 of 3...|progress: Done.|progress start: Timing run 2 of 3...|progress: Failed.", lines.join("|")); } - @TestData(1, "A test file is over its time.") - @TestData(2, "The test times were not read: their report exited with code 2.") + @TestData(3, "A test file is over its time.") + @TestData(1, "The test times were not read: their report exited with code 1.") + @TestData(2,"The test times were not read: their report exited with code 2.") @TestData(null, "The test times were not read: their report exited with code null.") @TestMethod public async failsForAFileOverItsTimeOrTimesNotReadAsync(exitCode: number | null, message: string): Promise { @@ -236,7 +242,7 @@ export class TimingStepTests { Assert.areEqual("_build/test-results.json holds no test times: the test run wrote none.\n", result.errorOutput); } - @TestData("[{\"file\":\"a\",\"ms\":200}]", "", 1, "timing a\nsummary 1 files 2 runs\n") + @TestData("[{\"file\":\"a\",\"ms\":200}]", "", 3, "timing a\nsummary 1 files 2 runs\n") @TestData("[{\"file\":\"a\",\"ms\":100}]", "", 0, "summary 1 files 2 runs\n") @TestData("[{\"file\":\"a\",\"ms\":100}]", "a\n", 0, "timing a\nsummary 1 files 2 runs\n") @TestMethod @@ -249,6 +255,39 @@ export class TimingStepTests { Assert.areEqual(output, result.output); } + @TestData("", "run-2.json holds no test times: the test run wrote none.\n") + @TestData("{", "run-2.json holds no test times: The file is not JSON.\n") + @TestMethod + public async failsToJudgeRunsWhoseTimesAreNotReadAsync(secondRun: string, errorOutput: string): Promise { + using checkout = new Checkout({ ...TimingStepTests.TESTING_PACKAGE, "run-1.json": "[{\"file\":\"a\",\"ms\":100}]", ...secondRun === "" ? {} : { "run-2.json": secondRun } }); + + const result = await new ProcessRunner().runAsync(process.execPath, [TimingStepTests.TIMES_ENTRY_POINT_PATH, "judge", "100", "0.5", "README.md", "run-1.json", "run-2.json"], checkout.directory); + + Assert.areEqual(2, result.exitCode); + Assert.areEqual("", result.output); + Assert.areEqual(errorOutput, result.errorOutput); + } + + @TestMethod + public async endsAFaultReadingTheTimesWithItsStackTraceAsync(): Promise { + using checkout = new Checkout({ ...TimingStepTests.TESTING_PACKAGE, "run-1.json": "fault" }); + + const result = await new ProcessRunner().runAsync(process.execPath, [TimingStepTests.TIMES_ENTRY_POINT_PATH, "judge", "100", "0.5", "README.md", "run-1.json"], checkout.directory); + + Assert.areEqual(1, result.exitCode); + Assert.isTrue(result.errorOutput.includes("TypeError: A fault."), result.errorOutput); + } + + @TestMethod + public async failsToPrintTheSlowestTestsOfATimesFileThatIsNotJsonAsync(): Promise { + using checkout = new Checkout({ ...TimingStepTests.TESTING_PACKAGE, "_build/test-results.json": "{" }); + + const result = await new ProcessRunner().runAsync(process.execPath, [TimingStepTests.TIMES_ENTRY_POINT_PATH, "slowest", "_build/test-results.json", "10"], checkout.directory); + + Assert.areEqual(2, result.exitCode); + Assert.areEqual("_build/test-results.json holds no test times: The file is not JSON.\n", result.errorOutput); + } + @TestMethod public async failsToJudgeWithoutTheTestingPackageAsync(): Promise { using checkout = new Checkout(); From 9a906fe4534af62032d89315affe43c87cbcb165 Mon Sep 17 00:00:00 2001 From: Rostislav Rotaru Date: Sat, 14 Nov 2026 15:46:36 -0800 Subject: [PATCH 3/4] Check sibling peer ranges in the archive step A package that takes its siblings as peers names each one in the range from the version to the next minor, such as >=0.0.9 <0.1.0. The archive step now checks that range for every expected package under peerDependencies, and fails naming the archive, the peer, the range and the one expected; exact dependencies keep their check. Version gains nextMinor() for the upper bound. Agent: DEV1 (developer) Agent-Model: Anthropic Opus 5.5, Medium --- src/noldova.tools.pipelines/README.md | 4 +- .../src/api/index.d.ts | 27 ++++++++-- .../src/models/version.ts | 5 ++ src/noldova.tools.pipelines/src/resources.ts | 8 +++ .../src/services/archive-step.ts | 9 ++++ .../tests/models/version.test.ts | 8 +++ .../tests/services/archive-step.test.ts | 51 +++++++++++++++++++ 7 files changed, 106 insertions(+), 6 deletions(-) diff --git a/src/noldova.tools.pipelines/README.md b/src/noldova.tools.pipelines/README.md index b2c2070..40b8bb0 100644 --- a/src/noldova.tools.pipelines/README.md +++ b/src/noldova.tools.pipelines/README.md @@ -75,7 +75,7 @@ It succeeds only when: - the version is a version, the commit is its full id, and the scope is `@` and a lowercase name, such as `@noldova`; - the checkout is at the commit; - the folder holds one archive for each package at the version, named as `npm pack` names it, such as `noldova-tools.pipelines-0.0.4.tgz`, and nothing else; -- each archive's `package/package.json` has the package's name, inside the scope, the version, and the commit as its `gitHead`, isn't `"private": true`, has a `license`, and lists any other expected package under `dependencies` at exactly the version; +- each archive's `package/package.json` has the package's name, inside the scope, the version, and the commit as its `gitHead`, isn't `"private": true`, has a `license`, lists any other expected package under `dependencies` at exactly the version, and lists any under `peerDependencies` in the range from the version to the next minor one, such as `>=0.0.9 <0.1.0`, as a package that takes its siblings as peers names them; - each archive holds `package/LICENSE`, and no file of a kind that holds credentials: `.npmrc`, `.netrc`, `.env` or `.env.`, `id_rsa`, `id_dsa`, `id_ecdsa`, `id_ed25519`, or a name ending in `.pem`, `.key`, `.p12` or `.pfx`, such as `.key` itself, in any folder and whatever its case; - each archive holds every file the build copies into its package, as a file with the same bytes as its source, and neither the copy nor its source is a symbolic link or under one. @@ -102,7 +102,7 @@ A wrong value, a checkout at another commit or a missing folder fails with that Otherwise the step fails with one line for each problem it finds with the archives, such as `archives has no noldova-tools.pipelines-0.0.4.tgz.` or `noldova-tools.pipelines-0.0.4.tgz: its version is 0.0.3, not 0.0.4.` It reads each manifest and lists each archive's files with the `tar` program, which must be on the `PATH`; an archive `tar` can't read or list is one of those lines, with `tar`'s exit code and message. When the archives pass, the step checks the copies, extracting each archive with `tar` into a new temporary folder that it removes after, and adds one line, such as `Each of the 2 files the build copies in matches its source.`; it fails with npm's exit code and message when the build can't list its copies, and when its output doesn't end with `end of copies`; otherwise with one line for each copy the build lists that isn't three fields, has a path that isn't inside its folder or a package without an archive, a source that is a symbolic link or under one or isn't a file, an archive `tar` can't extract, and a copy an archive lacks, holds as a symbolic link, under one or as a folder, or holds with other bytes. -A `package.json`, in an archive or under `src/`, makes the step throw `JsonException` if it isn't a JSON object, lacks a `name` or `version` text where the step needs one, has a `gitHead` or a `license` that isn't a text, or has a `private` that isn't `true` or `false` or `dependencies` that aren't an object of texts. +A `package.json`, in an archive or under `src/`, makes the step throw `JsonException` if it isn't a JSON object, lacks a `name` or `version` text where the step needs one, has a `gitHead` or a `license` that isn't a text, or has a `private` that isn't `true` or `false` or `dependencies` or `peerDependencies` that aren't an object of texts. ### The publish step diff --git a/src/noldova.tools.pipelines/src/api/index.d.ts b/src/noldova.tools.pipelines/src/api/index.d.ts index 01be51c..96e924a 100644 --- a/src/noldova.tools.pipelines/src/api/index.d.ts +++ b/src/noldova.tools.pipelines/src/api/index.d.ts @@ -262,6 +262,22 @@ export declare class Version { */ public equals(other: Version): boolean; + /** + * Gives the first release of the next minor version, the upper bound of a + * sibling package's peer range. + * + * @returns The version with the minor part one higher, the patch part 0 + * and no nightly date. + * @example + * ```ts + * import { Version } from "@noldova/tools.pipelines"; + * + * Version.parse("0.0.9").nextMinor().toString(); // "0.1.0" + * Version.parse("1.4.1-nightly.20261101").nextMinor().toString(); // "1.5.0" + * ``` + */ + public nextMinor(): Version; + /** * Writes the version as its text. * @@ -1943,8 +1959,10 @@ export declare class SourceStep implements IStep { * the commit, and the folder holds exactly one archive for each package at * the version, named as `npm pack` names it, whose `package/package.json` * has the package's name, inside the scope, the version, and the commit as - * its `gitHead`, is not `"private": true`, has a `license`, and lists any expected package under `dependencies` at - * exactly the version. Each archive must also hold `package/LICENSE`, and no + * its `gitHead`, is not `"private": true`, has a `license`, lists any + * expected package under `dependencies` at exactly the version, and lists any + * under `peerDependencies` in the range from the version to the next minor + * one, such as `>=0.0.9 <0.1.0`. Each archive must also hold `package/LICENSE`, and no * file of a kind that holds credentials: `.npmrc`, `.netrc`, `.env` or * `.env.`, `id_rsa`, `id_dsa`, `id_ecdsa`, `id_ed25519`, or a name * ending in `.pem`, `.key`, `.p12` or `.pfx`, such as `.key` itself, in any @@ -2002,8 +2020,9 @@ export declare class ArchiveStep implements IStep { * `tar` cannot read the manifest of, with `tar`'s exit code and message, * and a manifest with another name, a name outside the scope, another * version, no `gitHead` or another one, `"private": true`, no `license` - * or an empty one, or an expected package under `dependencies` - * at anything but exactly the version, an archive `tar` cannot list the + * or an empty one, an expected package under `dependencies` + * at anything but exactly the version, or one under `peerDependencies` in + * any range but `>= `, an archive `tar` cannot list the * files of, with `tar`'s exit code and message, a file of a kind that holds * credentials, and no `package/LICENSE`. When the archives pass, it asks * the build for its copies and adds a line, such as diff --git a/src/noldova.tools.pipelines/src/models/version.ts b/src/noldova.tools.pipelines/src/models/version.ts index 4ed7d3f..0988f6c 100644 --- a/src/noldova.tools.pipelines/src/models/version.ts +++ b/src/noldova.tools.pipelines/src/models/version.ts @@ -61,6 +61,11 @@ export class Version { return this.compareTo(other) === 0; } + /** The first release of the next minor version, such as 0.1.0 for 0.0.9 or 0.0.9-nightly.20261114. */ + public nextMinor(): Version { + return new Version(this.major, this.minor + 1, 0); + } + public toString(): string { const release = [this.major, this.minor, this.patch].join(Resources.versionPartSeparator); diff --git a/src/noldova.tools.pipelines/src/resources.ts b/src/noldova.tools.pipelines/src/resources.ts index 7e292bc..d94e7db 100644 --- a/src/noldova.tools.pipelines/src/resources.ts +++ b/src/noldova.tools.pipelines/src/resources.ts @@ -596,6 +596,14 @@ export class Resources { return `${archiveName}: its dependency ${dependency} is ${found}, not exactly ${version}.`; } + public static formatSiblingPeerRange(version: string, nextMinor: string): string { + return `>=${version} <${nextMinor}`; + } + + public static formatPeerRangeDiffers(archiveName: string, peer: string, found: string, expected: string): string { + return `${archiveName}: its peer ${peer} is ${found}, not ${expected}.`; + } + public static formatArchiveChecked(archiveName: string, packageName: string, version: string, commit: string): string { return `${archiveName}: ${packageName}@${version} from commit ${commit}`; } diff --git a/src/noldova.tools.pipelines/src/services/archive-step.ts b/src/noldova.tools.pipelines/src/services/archive-step.ts index e9f9487..92df7c8 100644 --- a/src/noldova.tools.pipelines/src/services/archive-step.ts +++ b/src/noldova.tools.pipelines/src/services/archive-step.ts @@ -147,6 +147,15 @@ export class ArchiveStep implements IStep { findings.push(Resources.formatDependencyDiffers(archiveName, dependency, range, version)); } } + if (manifest.hasField(Resources.peerDependenciesField)) { + const peers = manifest.readObject(Resources.peerDependenciesField); + const expected = Resources.formatSiblingPeerRange(version, Version.parse(version).nextMinor().toString()); + for (const peer of packageNames.filter(t => peers.hasField(t))) { + const range = peers.readString(peer); + if (range !== expected) + findings.push(Resources.formatPeerRangeDiffers(archiveName, peer, range, expected)); + } + } findings.push(...await this.checkFilesAsync(archivePath)); diff --git a/src/noldova.tools.pipelines/tests/models/version.test.ts b/src/noldova.tools.pipelines/tests/models/version.test.ts index 1b989f0..cbd3bde 100644 --- a/src/noldova.tools.pipelines/tests/models/version.test.ts +++ b/src/noldova.tools.pipelines/tests/models/version.test.ts @@ -26,6 +26,14 @@ export class VersionTests { Assert.areEqual(text, version.toString()); } + @TestData("0.0.9", "0.1.0") + @TestData("1.4.1", "1.5.0") + @TestData("1.4.1-nightly.20261101", "1.5.0") + @TestMethod + public givesTheFirstReleaseOfTheNextMinorVersion(text: string, nextMinor: string): void { + Assert.areEqual(nextMinor, Version.parse(text).nextMinor().toString()); + } + @TestData("1.4.1-nightly.20261101", 1, 4, 1, "20261101") @TestData("0.0.0-nightly.10000101", 0, 0, 0, "10000101") @TestData("0.1.1-nightly.20280229", 0, 1, 1, "20280229") diff --git a/src/noldova.tools.pipelines/tests/services/archive-step.test.ts b/src/noldova.tools.pipelines/tests/services/archive-step.test.ts index ad39179..5db7827 100644 --- a/src/noldova.tools.pipelines/tests/services/archive-step.test.ts +++ b/src/noldova.tools.pipelines/tests/services/archive-step.test.ts @@ -455,6 +455,45 @@ export class ArchiveStepTests { result.message); } + @TestMethod + public async succeedsForSiblingPeersInTheirRangeAsync(): Promise { + using repository = GitRepositoryFixture.create(); + const commit = repository.commit(ArchiveStepTests.PACKAGES); + const folder = ArchiveStepTests.packArchives(repository, ArchiveStepTests.MANIFEST_A, ArchiveStepTests.peerManifestB(">=0.0.4 <0.1.0")); + + const result = await ArchiveStepTests.createStep(repository.directory).runAsync([folder, "0.0.4", commit, "@noldova"]); + + Assert.isTrue(result.isSuccessful, result.message); + } + + @TestData(">=0.0.3 <0.1.0") + @TestData(">=0.0.4 <0.0.5") + @TestData(">=0.0.4 <1.0.0") + @TestData("0.0.4") + @TestData("^0.0.4") + @TestMethod + public async failsForASiblingPeerInAnotherRangeAsync(range: string): Promise { + using repository = GitRepositoryFixture.create(); + const commit = repository.commit(ArchiveStepTests.PACKAGES); + const folder = ArchiveStepTests.packArchives(repository, ArchiveStepTests.MANIFEST_A, ArchiveStepTests.peerManifestB(range)); + + const result = await ArchiveStepTests.createStep(repository.directory).runAsync([folder, "0.0.4", commit, "@noldova"]); + + Assert.isFalse(result.isSuccessful); + Assert.areEqual(`noldova-b-0.0.4.tgz: its peer @noldova/a is ${range}, not >=0.0.4 <0.1.0.`, result.message); + } + + @TestMethod + public async rejectsPeerDependenciesThatAreNotAnObjectAsync(): Promise { + using repository = GitRepositoryFixture.create(); + const commit = repository.commit(ArchiveStepTests.PACKAGES); + const folder = ArchiveStepTests.packArchives(repository, ArchiveStepTests.MANIFEST_A, ArchiveStepTests.MANIFEST_B.replace("\"dependencies\"", "\"peerDependencies\": [], \"dependencies\"")); + + const exception = await Assert.throwsAsync(() => ArchiveStepTests.createStep(repository.directory).runAsync([folder, "0.0.4", commit, "@noldova"]), JsonException); + + Assert.isTrue(exception.path.startsWith("noldova-b-0.0.4.tgz/package/package.json"), exception.path); + } + @TestMethod public async failsForAnArchiveFromAnotherCommitAsync(): Promise { using repository = GitRepositoryFixture.create(); @@ -700,6 +739,18 @@ export class ArchiveStepTests { return new ArchiveStep(new Git(processRunner, directory), new Tar(processRunner), new ListingNpm(directory, listed)); } + /** B's archived manifest taking its sibling @noldova/a as a peer in the range given, beside a third-party peer the step leaves alone. */ + private static peerManifestB(range: string): string { + return JSON.stringify({ + name: "@noldova/b", + version: "0.0.4", + gitHead: "__COMMIT__", + license: "MIT", + private: false, + peerDependencies: { "@noldova/a": range, "@example/d": "^1.0.0" } + }); + } + private static packArchives(repository: GitRepositoryFixture, manifestA: string, manifestB: string, filesA: Readonly> = {}): string { const commit = repository.readHead(); repository.pack("archives/noldova-a-0.0.4.tgz", { "package/package.json": ArchiveStepTests.withCommit(manifestA, commit), ...ArchiveStepTests.LICENSE, ...filesA }); From 2b12e63ebe7b18cdfe4084bae1b9610e2613685c Mon Sep 17 00:00:00 2001 From: Rostislav Rotaru Date: Sat, 14 Nov 2026 15:56:50 -0800 Subject: [PATCH 4/4] Drop the doc comments outside the declarations Version.nextMinor's name says what it gives, and the test helper's manifest reads plainly, so neither keeps a comment. Agent: DEV1 (developer) Agent-Model: Anthropic Opus 5.5, Medium --- src/noldova.tools.pipelines/src/models/version.ts | 1 - src/noldova.tools.pipelines/tests/services/archive-step.test.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/noldova.tools.pipelines/src/models/version.ts b/src/noldova.tools.pipelines/src/models/version.ts index 0988f6c..4a92a3d 100644 --- a/src/noldova.tools.pipelines/src/models/version.ts +++ b/src/noldova.tools.pipelines/src/models/version.ts @@ -61,7 +61,6 @@ export class Version { return this.compareTo(other) === 0; } - /** The first release of the next minor version, such as 0.1.0 for 0.0.9 or 0.0.9-nightly.20261114. */ public nextMinor(): Version { return new Version(this.major, this.minor + 1, 0); } diff --git a/src/noldova.tools.pipelines/tests/services/archive-step.test.ts b/src/noldova.tools.pipelines/tests/services/archive-step.test.ts index 5db7827..bad93ca 100644 --- a/src/noldova.tools.pipelines/tests/services/archive-step.test.ts +++ b/src/noldova.tools.pipelines/tests/services/archive-step.test.ts @@ -739,7 +739,6 @@ export class ArchiveStepTests { return new ArchiveStep(new Git(processRunner, directory), new Tar(processRunner), new ListingNpm(directory, listed)); } - /** B's archived manifest taking its sibling @noldova/a as a peer in the range given, beside a third-party peer the step leaves alone. */ private static peerManifestB(range: string): string { return JSON.stringify({ name: "@noldova/b",