diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6ae464..8efbbce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,6 +196,44 @@ jobs: shell: pwsh run: ./scripts/package-smoke.ps1 + differential: + name: Cross-language differential + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 22.19.0 + cache: npm + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: stable + cache-dependency-path: ports/go/go.sum + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: ports/python/pyproject.toml + - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5 + with: + dotnet-version: | + 8.0.x + 10.0.x + - name: Use supported npm + run: npm install --global npm@10.9.7 + - run: npm ci + - name: Install the Python port + run: python -m pip install ./ports/python + - name: Download Go modules + working-directory: ports/go + run: go mod download + - name: Restore .NET with locked dependencies + working-directory: ports/dotnet + run: dotnet restore --locked-mode + - name: Compare every implementation over one corpus + run: npm run differential + required: name: Required checks if: always() @@ -207,6 +245,7 @@ jobs: - python-package - dotnet - dotnet-package + - differential runs-on: ubuntu-latest steps: - name: Require successful CI @@ -218,6 +257,7 @@ jobs: PYTHON_PACKAGE_RESULT: ${{ needs.python-package.result }} DOTNET_RESULT: ${{ needs.dotnet.result }} DOTNET_PACKAGE_RESULT: ${{ needs.dotnet-package.result }} + DIFFERENTIAL_RESULT: ${{ needs.differential.result }} run: | test "$NODE_RESULT" = "success" test "$BENCHMARK_RESULT" = "success" @@ -226,3 +266,4 @@ jobs: test "$PYTHON_PACKAGE_RESULT" = "success" test "$DOTNET_RESULT" = "success" test "$DOTNET_PACKAGE_RESULT" = "success" + test "$DIFFERENTIAL_RESULT" = "success" diff --git a/CHANGELOG.md b/CHANGELOG.md index ae62084..013a53b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## Unreleased + +- Added a required cross-language differential CI gate that runs the TypeScript, + Go, Python, and .NET command-line tools over one shared corpus of golden, + edge, invalid, malformed, and seeded random inputs and compares their complete + verification results, error codes, and verification-record digests + (`npm run differential`, documented in `docs/DIFFERENTIAL.md`). +- Fixed the TypeScript CLI accepting transport bytes that are not valid UTF-8. + Node's lossy decoding replaced malformed bytes with U+FFFD and then verified + the corrupted evidence; the Go, Python, and .NET ports already rejected it. + The CLI now reports `WORLDCUT_INVALID_JSON`, and a byte-order mark stays + rejected. +- Hardened the Go port so a JSON number that underflows the IEEE-754 double + range, such as `1e-400`, is accepted as a finite zero like the other ports, + while syntax errors and overflow to infinity are still rejected. + ## 0.2.0 - 2026-09-03 - Added language-neutral protocol, canonicalization, and conformance diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f426bfe..62df889 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,16 @@ Before proposing a release-affecting change: npm run release:check ``` +Changes to protocol semantics, canonicalization, conformance data, or any +language port must also pass the four-toolchain differential gate: + +```sh +npm run differential +``` + +Its Go, Python, and .NET prerequisites and reproducible seed controls are +documented in [`docs/DIFFERENTIAL.md`](docs/DIFFERENTIAL.md). + ## Pull requests Keep changes focused and include tests for observable behavior. Protocol diff --git a/README.md b/README.md index 59aa17f..891f0ae 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,11 @@ Demonstrated package, GitHub integration, and benchmark evidence is summarized in [`docs/VALIDATION.md`](docs/VALIDATION.md). Language-neutral protocol semantics and golden vectors are under -[`spec/0.1`](spec/0.1) and [`conformance/0.1`](conformance/0.1). +[`spec/0.1`](spec/0.1) and [`conformance/0.1`](conformance/0.1). A required CI +gate additionally runs all four implementations over one shared corpus of golden, +edge, invalid, malformed, and seeded random inputs and compares their complete +results, as described in +[`docs/DIFFERENTIAL.md`](docs/DIFFERENTIAL.md). ## Implementations @@ -57,6 +61,14 @@ Language-neutral protocol semantics and golden vectors are under | [Python](ports/python) | 0.1 / 0.1.2 | Independent conformant verifier and CLI; integrations not yet included | | [.NET](ports/dotnet) | 0.1 / 0.1.2 | Independent conformant verifier and CLI for .NET 8 and .NET 10; integrations not yet included | +Every port passes the committed vectors, and the +[cross-language differential suite](docs/DIFFERENTIAL.md) checks that they still +agree with the TypeScript reference on inputs no vector covers: + +```sh +npm run differential +``` + ## Run the examples ```sh @@ -470,6 +482,23 @@ npm run benchmark The project uses the Node.js test runner and has no runtime dependencies. +The conformance corpus is checked with Node.js: + +```sh +npm run conformance:check +``` + +Cross-language work additionally needs Go, Python, and .NET toolchains: + +```sh +npm run differential +``` + +`npm run differential` builds each port's CLI once and compares all four +implementations over one shared corpus. Its seed, case count, and executable +overrides are documented in +[`docs/DIFFERENTIAL.md`](docs/DIFFERENTIAL.md). + Protocol details and runtime assumptions are documented in [`docs/PROTOCOL.md`](docs/PROTOCOL.md). Production deployment requirements are documented in [`docs/PRODUCTION.md`](docs/PRODUCTION.md). diff --git a/docs/DIFFERENTIAL.md b/docs/DIFFERENTIAL.md new file mode 100644 index 0000000..eac9acf --- /dev/null +++ b/docs/DIFFERENTIAL.md @@ -0,0 +1,203 @@ +# Cross-language differential verification + +The four WorldCut implementations share one protocol but no code. Passing the +committed vectors in `conformance/0.1` proves that each one agrees with the +specification on a small fixed corpus. It does not prove that they still agree +on inputs nobody wrote a vector for. + +The differential suite closes that gap. It runs the TypeScript, Go, Python, and +.NET command-line tools over one identical corpus of transport bytes and +compares their complete parsed verification results. + +```sh +npm run differential +``` + +The suite is a required CI check. The `differential` job in +[`.github/workflows/ci.yml`](../.github/workflows/ci.yml) runs it on every pull +request and `Required checks` fails unless it succeeds. + +## What is compared + +For every case the harness spawns each CLI with the same input file, then +classifies the outcome as a verification result, a stable error envelope, or an +unusable output. An unusable output always fails the case. + +| Case category | Comparison | +| --- | --- | +| Verification input that TypeScript accepts | Every port must print a result that is structurally identical to the TypeScript result, including summaries, `details`, acquisition action identifiers, costs and order, the acquisition plan, coverage counters, protocol and engine versions, and `verificationRecordDigest`. | +| Committed verification vector | The TypeScript result must additionally equal the committed `expected` result. | +| Verification input that TypeScript rejects | Every port must fail with the same stable `WORLDCUT_*` code. | +| Committed invalid vector | Every port must fail with the code recorded in `invalid-vectors.json`. | +| Malformed raw transport bytes | Every port must fail with an outcome that `spec/0.1/CONFORMANCE.md` permits: `PARSE_ERROR` or `WORLDCUT_INVALID_INPUT`. A port that accepts the bytes fails the case. | + +`verificationRecordDigest` is checked twice: it must match +`^[0-9a-f]{64}$` in every port, and every port's digest must equal the +TypeScript digest. Case pairs that are canonically identical but textually +different — reordered members, reordered observations and requirements, `\u` +escapes versus literal UTF-8 — must also produce the same digest as each other. + +## What is deliberately ignored + +Results are compared after `JSON.parse`, so the following never fail a case: + +- indentation, spacing, and line endings; +- object member order; +- `\uXXXX` escaping versus literal UTF-8; +- number spelling, for example `1e+21` versus `1E+21` versus `1000000...`; +- `-0` versus `0`, because `worldcut-json-v1` serializes negative zero as `0` + and the two spellings can never produce different digests. + +Everything else is treated as a semantic difference, including a missing member, +a different array order, a different summary string, and a different cost. + +## Case categories + +| Category | Source | Count | +| --- | --- | ---: | +| `golden` | every case in `conformance/0.1/verification-vectors.json` | 15 | +| `invalid` | every case in `conformance/0.1/invalid-vectors.json` | 12 | +| `raw` | every case in `conformance/0.1/raw-vectors.json` | 1 | +| `example` | every published fixture in `examples/` | 4 | +| `edge` | handcrafted deterministic cases | 31 | +| `transport` | handcrafted malformed byte sequences | 20 | +| `random` | seeded generated inputs | `--count`, default 500 | + +The `edge` category covers finite IEEE-754 underflow (`1e-400`), negative zero, +alternative number spellings, boundary doubles, integer precision loss, UTF-16 +member ordering across ASCII, Latin-1, full-width, and astral names, whitespace +and array-index value paths, structural `value_equals` comparison, deep nesting, +every dependency and temporal outcome, acquisition action de-duplication, +required and advisory aggregation, cost boundaries, and input array reordering. + +The `transport` category covers empty and whitespace-only files, truncated +documents, trailing values, trailing commas, byte-order marks, invalid UTF-8, +raw control characters, NUL bytes, unpaired surrogate escapes, `NaN` and +`Infinity` literals, numbers that overflow to infinity, leading zeros, +hexadecimal, single quotes, and non-object top-level values. + +Cases whose meaning depends on lexical form are authored as text or bytes, never +as JavaScript values. `JSON.stringify` would turn `1e-400` into `0` and `-0` +into `0` before any CLI could observe them. + +### Randomized inputs + +Randomized cases are generated from `${seed}:${index}` with a seeded sfc32 +generator, so a seed and count always reproduce the same bytes on every +platform. They exercise nested arrays and objects, Unicode strings and member +names, UTF-16 ordering, safe finite doubles and integers, negative zero, +`value_equals` hits and misses, satisfied, violated, and unknown dependency +cases, temporal overlap and gap cases, reordered observation and requirement +arrays, acquisition planning and de-duplication, and required and advisory +aggregation. Nesting stays far below the 48-level transport cap documented by +the .NET port. + +About a third of the generated corpus is built in a coherent mode where every +requirement is satisfiable, so the suite keeps reaching `CONTRACT_SATISFIED` and +`NOT_NEEDED` acquisition plans rather than only failure paths. The run fails if +fewer than 75% of randomized cases produce a verification result, or if the +randomized corpus stops reaching all three verdicts. + +## Options + +| Flag | Environment variable | Default | +| --- | --- | --- | +| `--seed ` | `WORLDCUT_DIFFERENTIAL_SEED` | `worldcut-0.1` | +| `--count ` | `WORLDCUT_DIFFERENTIAL_COUNT` | `500` | +| `--jobs ` | `WORLDCUT_DIFFERENTIAL_JOBS` | `max(2, min(6, available parallelism))` | +| `--only ` | — | all cases | +| `--category ` | — | all categories | +| `--max-failures ` | — | `10` reported in full | +| `--timeout-ms ` | `WORLDCUT_DIFFERENTIAL_TIMEOUT_MS` | `60000` per CLI invocation | +| `--list` | — | print the corpus and exit | +| `--self-check-only` | — | run the harness self-checks and exit | + +```sh +npm run differential -- --seed release-audit --count 2000 +npm run differential -- --only edge/number-underflow-positive --count 0 +npm run differential -- --category transport --count 0 --list +``` + +The harness runs deterministic self-checks before it starts any port. Those +checks cover the seeded generator, the raw-lexeme writer, the structural +comparison, the outcome mapping, and corpus invariants, so a broken harness +fails loudly instead of comparing a weaker corpus. `npm test` runs them too, so +a harness regression is caught by the Node job even though that job has no Go, +Python, or .NET toolchain. + +## Prerequisites + +| Port | Requirement | +| --- | --- | +| TypeScript | Node.js 22.19 or newer. `npm run differential` builds `dist/` first. | +| Go | A Go toolchain that satisfies `ports/go/go.mod`. The harness builds `cmd/worldcut-go` once. | +| Python | Python 3.11 or newer with the `ports/python` package installed, for example `python -m pip install ./ports/python`. | +| .NET | The .NET SDKs named in `ports/dotnet/global.json`. The harness builds `WorldCut.Tool` once. | + +Executables are resolved from `PATH` unless overridden: + +| Variable | Default | Purpose | +| --- | --- | --- | +| `WORLDCUT_NODE` | the running `node` | TypeScript CLI host | +| `WORLDCUT_GO` | `go` | Go toolchain | +| `WORLDCUT_PYTHON` | `python` | Python interpreter that can import `worldcut` | +| `WORLDCUT_DOTNET` | `dotnet` | .NET host | +| `WORLDCUT_DOTNET_FRAMEWORK` | `net8.0` | .NET target framework to build and run | + +Example on a machine with private toolchains: + +```sh +WORLDCUT_GO=/opt/go/bin/go \ +WORLDCUT_PYTHON=ports/python/.venv/bin/python \ +WORLDCUT_DOTNET=/opt/dotnet/dotnet \ +npm run differential +``` + +Inputs are written to a temporary directory outside the repository and that +directory is always removed, including after a failure. + +Each CLI invocation is terminated if it exceeds the configured timeout. A +timeout, runtime error, argument error, or file error never counts as an +allowed parser rejection for malformed transport bytes. + +## Reading a failure + +A failing case prints its identifier and category, the seed and count that +produced it, a ready-to-paste reproduction command, the exact input bytes with +their length and SHA-256, and the specific differences per port as JSON +pointers. Inputs larger than 8 KiB, or inputs that are not valid UTF-8, are +printed as base64 so the exact bytes survive. Nothing else from the workspace is +printed. + +## Why TypeScript is the oracle + +The TypeScript package is the reference implementation. `spec/0.1` and the +committed vectors are generated from it by `npm run conformance:update`, and +`spec/0.1/CANONICALIZATION.md` defines canonicalization in terms of the +ECMAScript rules it follows. When the ports disagree, TypeScript defines the +answer unless TypeScript itself is shown to violate the specification, in which +case the specification, the vectors, and every implementation are corrected +together. + +Using one oracle is a pragmatic choice, not a claim that TypeScript is correct. +The suite also re-checks TypeScript against the committed golden results on +every run, so a regression in the oracle fails the gate instead of being copied +into the other ports. + +## What this does and does not establish + +Agreement across a large shared corpus is evidence of protocol equivalence for +the inputs that were compared. It is not a proof. + +The suite does not: + +- exhaustively cover the input space, or replace the committed vectors; +- prove the four implementations are equivalent for untested inputs; +- constitute a formal verification, a model check, or a refinement proof; +- compare library APIs, only the command-line tools; +- exercise every supported language runtime version — CI runs the differential + job on one Node.js, Go, Python, and .NET version each, while the independent + per-language jobs cover the full support matrix. + +Its value is regression pressure: an accidental divergence introduced by a +change to any one port is very likely to fail this gate before it is released. diff --git a/package.json b/package.json index e445f05..6597859 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "docs/INTEGRATIONS.md", "docs/AGENTIC_DATA_KERNEL.md", "docs/VALIDATION.md", + "docs/DIFFERENTIAL.md", "spec", "conformance", "README.md", @@ -85,12 +86,13 @@ "clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"", "build": "npm run clean && tsc -p tsconfig.json", "prepack": "npm run build", - "check": "tsc -p tsconfig.json --noEmit", + "check": "tsc -p tsconfig.json --noEmit && tsc -p scripts/differential/jsconfig.json", "test": "npm run build && node --test \"dist/test/*.test.js\"", "test:package": "node scripts/test-package.mjs", "release:check": "npm run check && npm test && npm run examples && npm run benchmark && npm run test:package", "conformance:update": "npm run build && node scripts/generate-conformance.mjs --write", "conformance:check": "npm run build && node scripts/generate-conformance.mjs", + "differential": "npm run conformance:check && node scripts/differential.mjs", "example": "npm run examples", "examples": "npm run build && node dist/examples/fixtures.js", "verify": "npm run build && node dist/cli.js", diff --git a/ports/go/numbers_test.go b/ports/go/numbers_test.go new file mode 100644 index 0000000..27408fb --- /dev/null +++ b/ports/go/numbers_test.go @@ -0,0 +1,142 @@ +package worldcut + +import ( + "math" + "strings" + "testing" +) + +// TestDecodeAcceptsFiniteUnderflow locks the behaviour that a JSON number which +// underflows the IEEE-754 double range is a finite zero rather than a protocol +// error. TypeScript is normative here: JSON.parse("1e-400") is 0, and the +// Python and .NET ports agree. +func TestDecodeAcceptsFiniteUnderflow(t *testing.T) { + for _, testCase := range []struct { + lexeme string + negative bool + }{ + {"1e-400", false}, + {"-1e-400", true}, + {"1e-1000", false}, + {"-1e-1000", true}, + {"5e-324", false}, + {"-5e-324", true}, + {"0.00000000000000000000000000000000000000000000000000000000001e-400", false}, + } { + t.Run(testCase.lexeme, func(t *testing.T) { + decoded, err := decodeJSON([]byte(testCase.lexeme)) + if err != nil { + t.Fatalf("decodeJSON(%q) = %v", testCase.lexeme, err) + } + number, ok := decoded.(float64) + if !ok { + t.Fatalf("decodeJSON(%q) produced %T", testCase.lexeme, decoded) + } + if math.IsInf(number, 0) || math.IsNaN(number) { + t.Fatalf("decodeJSON(%q) = %v, want a finite value", testCase.lexeme, number) + } + if testCase.negative && !math.Signbit(number) { + t.Fatalf("decodeJSON(%q) = %v, want a negative value", testCase.lexeme, number) + } + if !testCase.negative && math.Signbit(number) { + t.Fatalf("decodeJSON(%q) = %v, want a non-negative value", testCase.lexeme, number) + } + }) + } +} + +// TestDecodeRejectsOverflowAndSyntaxErrors keeps the underflow allowance from +// widening into acceptance of infinities or malformed numbers. +func TestDecodeRejectsOverflowAndSyntaxErrors(t *testing.T) { + for _, lexeme := range []string{ + "1e400", + "-1e400", + "1e309", + "1e99999999999999999999", + "01", + "1e", + "0x1", + "NaN", + "Infinity", + "-Infinity", + ".5", + "1.", + } { + t.Run(lexeme, func(t *testing.T) { + if _, err := decodeJSON([]byte(lexeme)); err == nil { + t.Fatalf("decodeJSON(%q) unexpectedly succeeded", lexeme) + } + }) + } +} + +// TestVerifyAcceptsUnderflowInsideAnObservation proves the allowance reaches the +// public API and that an underflowing lexeme compares equal to zero. +func TestVerifyAcceptsUnderflowInsideAnObservation(t *testing.T) { + template := `{ + "protocolVersion":"0.1", + "contract":{ + "id":"underflow","version":"1","decisionTime":"2026-01-01T00:00:00.000Z", + "assumptions":{"clockModel":"trusted_normalized","intervalModel":"half_open","metadataModel":"honest_but_possibly_incomplete"}, + "requirements":[ + {"id":"tiny","description":"Tiny","type":"value_equals","role":"present","path":["tiny"],"expected":EXPECTED} + ] + }, + "observations":[{ + "id":"observation","role":"present", + "resource":{"provider":"p","account":"a","kind":"k","key":"x"}, + "value":{"tiny":LEXEME},"observedAt":"2026-01-01T00:00:00.000Z","acquisitionCost":1, + "witness":{"provenance":"provider_asserted"} + }] + }` + + for _, testCase := range []struct { + name string + lexeme string + expected string + verdict string + }{ + {"positive-underflow-equals-zero", "1e-400", "0", "CONTRACT_SATISFIED"}, + {"negative-underflow-equals-zero", "-1e-400", "0", "CONTRACT_SATISFIED"}, + {"negative-underflow-equals-negative-zero", "-1e-400", "-0", "CONTRACT_SATISFIED"}, + {"underflow-is-not-one", "1e-400", "1", "CONTRACT_VIOLATED"}, + } { + t.Run(testCase.name, func(t *testing.T) { + source := strings.ReplaceAll(template, "LEXEME", testCase.lexeme) + source = strings.ReplaceAll(source, "EXPECTED", testCase.expected) + result, err := VerifyJSON([]byte(source)) + if err != nil { + t.Fatalf("VerifyJSON = %v", err) + } + if result.Verdict != testCase.verdict { + t.Fatalf("verdict = %s, want %s", result.Verdict, testCase.verdict) + } + }) + } +} + +// TestVerifyRejectsOverflowInsideAnObservation keeps overflow failing closed +// through the public API. +func TestVerifyRejectsOverflowInsideAnObservation(t *testing.T) { + source := `{ + "protocolVersion":"0.1", + "contract":{ + "id":"overflow","version":"1","decisionTime":"2026-01-01T00:00:00.000Z", + "assumptions":{"clockModel":"trusted_normalized","intervalModel":"half_open","metadataModel":"honest_but_possibly_incomplete"}, + "requirements":[ + {"id":"big","description":"Big","type":"value_equals","role":"present","path":["big"],"expected":1} + ] + }, + "observations":[{ + "id":"observation","role":"present", + "resource":{"provider":"p","account":"a","kind":"k","key":"x"}, + "value":{"big":1e400},"observedAt":"2026-01-01T00:00:00.000Z","acquisitionCost":1, + "witness":{"provenance":"provider_asserted"} + }] + }` + if _, err := VerifyJSON([]byte(source)); err == nil { + t.Fatal("an overflowing JSON number was accepted") + } else if code := ErrorCode(err); code != InvalidInputCode { + t.Fatalf("error code = %q, want %q", code, InvalidInputCode) + } +} diff --git a/ports/go/validation.go b/ports/go/validation.go index 33f856e..4baf268 100644 --- a/ports/go/validation.go +++ b/ports/go/validation.go @@ -94,7 +94,14 @@ func normalizeNumbers(value any) (any, error) { switch typed := value.(type) { case json.Number: number, err := strconv.ParseFloat(string(typed), 64) - if err != nil || math.IsInf(number, 0) || math.IsNaN(number) { + // strconv.ErrRange covers both IEEE-754 underflow, which yields a + // finite +/-0, and overflow, which yields +/-Inf. TypeScript, Python, + // and .NET all accept underflow such as 1e-400 as zero, so only the + // non-finite results are protocol errors here. + if err != nil && !errors.Is(err, strconv.ErrRange) { + return nil, fmt.Errorf("invalid JSON number %q", typed) + } + if math.IsInf(number, 0) || math.IsNaN(number) { return nil, fmt.Errorf("invalid JSON number %q", typed) } return number, nil diff --git a/scripts/differential.mjs b/scripts/differential.mjs new file mode 100644 index 0000000..240740d --- /dev/null +++ b/scripts/differential.mjs @@ -0,0 +1,645 @@ +#!/usr/bin/env node +/** + * Cross-language differential verification. + * + * Runs the TypeScript, Go, Python, and .NET WorldCut CLIs over one identical + * corpus of transport bytes and compares their complete parsed verification + * results. TypeScript is the oracle: every other port must reproduce its result + * exactly, or fail with the same stable error code, or - for malformed + * transport bytes - fail with an outcome that `spec/0.1/CONFORMANCE.md` + * explicitly permits. + * + * Usage: + * node scripts/differential.mjs [--seed ] [--count ] [--only ] + * [--category ] [--jobs ] + * [--max-failures ] [--timeout-ms ] + * [--list] [--self-check-only] + * + * Environment overrides: + * WORLDCUT_GO, WORLDCUT_PYTHON, WORLDCUT_DOTNET, WORLDCUT_NODE, + * WORLDCUT_DOTNET_FRAMEWORK, WORLDCUT_DIFFERENTIAL_SEED, + * WORLDCUT_DIFFERENTIAL_COUNT, WORLDCUT_DIFFERENTIAL_JOBS, + * WORLDCUT_DIFFERENTIAL_TIMEOUT_MS + */ + +import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { availableParallelism, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + deterministicCases, + selectCases, +} from "./differential/corpus.mjs"; +import { generateRandomCases } from "./differential/generate.mjs"; +import { + DIGEST_PATTERN, + diffJson, + preview, + toRawOutcome, +} from "./differential/compare.mjs"; +import { prepareRunners, runPort } from "./differential/ports.mjs"; +import { runSelfChecks } from "./differential/self-check.mjs"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +/** The seed used by CI and by every documented reproduction command. */ +const DEFAULT_SEED = "worldcut-0.1"; + +/** The randomized case count used by CI. */ +const DEFAULT_COUNT = 500; + +/** Fraction of randomized cases that must verify successfully. */ +const MIN_RANDOM_SUCCESS_RATIO = 0.75; + +/** + * @param {string} raw + * @param {string} name + * @param {number} minimum + * @returns {number} + */ +function parseIntegerOption(raw, name, minimum) { + if (!/^(0|[1-9]\d*)$/.test(raw)) { + throw new Error(`${name} must be an integer, got ${raw}`); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < minimum) { + throw new Error( + `${name} must be an integer of at least ${minimum}, got ${raw}`, + ); + } + return value; +} + +/** + * @param {string[]} argv + * @returns {{ + * seed: string, + * count: number, + * only: string | null, + * category: string | null, + * jobs: number, + * list: boolean, + * selfCheckOnly: boolean, + * maxFailures: number, + * timeoutMs: number, + * }} + */ +function parseArguments(argv) { + /** @type {Record} */ + const flags = {}; + const booleans = new Set(); + const valueFlags = new Set([ + "seed", + "count", + "only", + "category", + "jobs", + "max-failures", + "timeout-ms", + ]); + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === undefined || !argument.startsWith("--")) { + throw new Error(`unexpected argument: ${String(argument)}`); + } + const name = argument.slice(2); + if (name === "list" || name === "self-check-only") { + booleans.add(name); + continue; + } + if (!valueFlags.has(name)) { + throw new Error(`unknown option: --${name}`); + } + const value = argv[index + 1]; + if (value === undefined) { + throw new Error(`--${name} requires a value`); + } + flags[name] = value; + index += 1; + } + + const seed = + flags["seed"] ?? process.env["WORLDCUT_DIFFERENTIAL_SEED"] ?? DEFAULT_SEED; + const rawCount = + flags["count"] ?? + process.env["WORLDCUT_DIFFERENTIAL_COUNT"] ?? + String(DEFAULT_COUNT); + const count = parseIntegerOption(rawCount, "--count", 0); + const rawJobs = + flags["jobs"] ?? + process.env["WORLDCUT_DIFFERENTIAL_JOBS"] ?? + String(Math.max(2, Math.min(6, availableParallelism()))); + const jobs = parseIntegerOption(rawJobs, "--jobs", 1); + const rawMaxFailures = flags["max-failures"] ?? "10"; + const maxFailures = parseIntegerOption( + rawMaxFailures, + "--max-failures", + 1, + ); + const rawTimeoutMs = + flags["timeout-ms"] ?? + process.env["WORLDCUT_DIFFERENTIAL_TIMEOUT_MS"] ?? + "60000"; + const timeoutMs = parseIntegerOption(rawTimeoutMs, "--timeout-ms", 1000); + + return { + seed, + count, + only: flags["only"] ?? null, + category: flags["category"] ?? null, + jobs, + list: booleans.has("list"), + selfCheckOnly: booleans.has("self-check-only"), + maxFailures, + timeoutMs, + }; +} + +/** + * Renders transport bytes for a failure report without leaking anything else. + * + * @param {Buffer} bytes + * @returns {string} + */ +function describeInput(bytes) { + const digest = createHash("sha256").update(bytes).digest("hex"); + const header = ` bytes: ${bytes.length}, sha256: ${digest}`; + const text = bytes.toString("utf8"); + const roundTrips = Buffer.from(text, "utf8").equals(bytes); + if (roundTrips && bytes.length <= 8192) { + return `${header}\n input: ${JSON.stringify(text)}`; + } + return `${header}\n input(base64): ${bytes.toString("base64")}`; +} + +/** + * @param {import("./differential/ports.mjs").PortOutcome} outcome + * @returns {string} + */ +function describeOutcome(outcome) { + if (outcome.kind === "result") { + const digest = /** @type {{ verificationRecordDigest?: unknown }} */ ( + outcome.result + )?.verificationRecordDigest; + return `exit 0, digest ${String(digest)}`; + } + if (outcome.kind === "error") { + return `exit ${outcome.status}, ${outcome.code}: ${preview(outcome.message, 120)}`; + } + return `exit ${outcome.status}, ${outcome.failure ?? "unusable output"}`; +} + +/** + * Compares one case across every port. + * + * @param {import("./differential/corpus.mjs").DifferentialCase} testCase + * @param {import("./differential/ports.mjs").PortRunner[]} runners + * @param {Map} outcomes + * @returns {string[]} Problem descriptions; empty when the case agrees. + */ +function compareCase(testCase, runners, outcomes) { + /** @type {string[]} */ + const problems = []; + + for (const runner of runners) { + const outcome = outcomes.get(runner.id); + if (outcome === undefined) { + problems.push(`${runner.id}: produced no outcome`); + continue; + } + if (outcome.kind === "unusable") { + problems.push(`${runner.id}: ${outcome.failure}`); + } + } + if (problems.length > 0) { + return problems; + } + + if (testCase.expect === "transport") { + const allowed = testCase.allowed ?? []; + for (const runner of runners) { + const outcome = /** @type {import("./differential/ports.mjs").PortOutcome} */ ( + outcomes.get(runner.id) + ); + if (outcome.kind !== "error") { + problems.push( + `${runner.id}: accepted malformed transport bytes (${describeOutcome(outcome)})`, + ); + continue; + } + const observed = toRawOutcome(outcome.code ?? ""); + if (observed === null || !allowed.includes(observed)) { + problems.push( + `${runner.id}: error code ${outcome.code} is not an allowed transport rejection (${allowed.join(", ")})`, + ); + } + } + return problems; + } + + if (testCase.expect === "code") { + for (const runner of runners) { + const outcome = /** @type {import("./differential/ports.mjs").PortOutcome} */ ( + outcomes.get(runner.id) + ); + if (outcome.kind !== "error") { + problems.push( + `${runner.id}: accepted an invalid input (${describeOutcome(outcome)})`, + ); + continue; + } + if (outcome.code !== testCase.code) { + problems.push( + `${runner.id}: error code ${outcome.code}, expected ${testCase.code}`, + ); + } + } + return problems; + } + + const oracle = /** @type {import("./differential/ports.mjs").PortOutcome} */ ( + outcomes.get("typescript") + ); + + if (testCase.expect === "result" && oracle.kind !== "result") { + problems.push( + `typescript: expected a verification result but got ${describeOutcome(oracle)}`, + ); + return problems; + } + + if (oracle.kind === "error") { + for (const runner of runners) { + if (runner.id === "typescript") { + continue; + } + const outcome = /** @type {import("./differential/ports.mjs").PortOutcome} */ ( + outcomes.get(runner.id) + ); + if (outcome.kind !== "error") { + problems.push( + `${runner.id}: accepted an input TypeScript rejected with ${oracle.code}`, + ); + continue; + } + if (outcome.code !== oracle.code) { + problems.push( + `${runner.id}: error code ${outcome.code}, TypeScript reported ${oracle.code}`, + ); + } + } + return problems; + } + + const oracleResult = /** @type {Record} */ (oracle.result); + const oracleDigest = oracleResult["verificationRecordDigest"]; + if (typeof oracleDigest !== "string" || !DIGEST_PATTERN.test(oracleDigest)) { + problems.push( + `typescript: verificationRecordDigest ${preview(oracleDigest)} is not 64 lowercase hex characters`, + ); + } + + if (testCase.expectedResult !== undefined) { + const golden = diffJson(testCase.expectedResult, oracleResult, { + limit: 5, + }); + for (const difference of golden) { + problems.push( + `typescript: committed vector mismatch at ${difference.path || "/"} (${difference.reason}): expected ${preview(difference.expected)}, got ${preview(difference.actual)}`, + ); + } + } + + for (const runner of runners) { + if (runner.id === "typescript") { + continue; + } + const outcome = /** @type {import("./differential/ports.mjs").PortOutcome} */ ( + outcomes.get(runner.id) + ); + if (outcome.kind !== "result") { + problems.push( + `${runner.id}: TypeScript verified this input but the port reported ${describeOutcome(outcome)}`, + ); + continue; + } + const portResult = /** @type {Record} */ (outcome.result); + const portDigest = portResult["verificationRecordDigest"]; + if (typeof portDigest !== "string" || !DIGEST_PATTERN.test(portDigest)) { + problems.push( + `${runner.id}: verificationRecordDigest ${preview(portDigest)} is not 64 lowercase hex characters`, + ); + } else if (portDigest !== oracleDigest) { + problems.push( + `${runner.id}: verificationRecordDigest ${portDigest} != ${String(oracleDigest)}`, + ); + } + for (const difference of diffJson(oracleResult, portResult, { limit: 8 })) { + problems.push( + `${runner.id}: ${difference.path || "/"} (${difference.reason}): TypeScript ${preview(difference.expected)}, port ${preview(difference.actual)}`, + ); + } + } + + return problems; +} + +/** + * Renders a tally as a stable, human-readable summary. + * + * @param {Map} counts + * @returns {string} + */ +function describeCounts(counts) { + if (counts.size === 0) { + return "none"; + } + return [...counts.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, total]) => `${name}=${total}`) + .join(" "); +} + +/** + * Runs an async worker over a list with bounded concurrency, preserving order. + * + * @template T + * @template R + * @param {readonly T[]} items + * @param {number} limit + * @param {(item: T, index: number) => Promise} worker + * @returns {Promise} + */ +async function mapWithLimit(items, limit, worker) { + /** @type {R[]} */ + const results = new Array(items.length); + let next = 0; + const runners = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (true) { + const index = next; + next += 1; + if (index >= items.length) { + return; + } + const item = items[index]; + if (item === undefined) { + return; + } + results[index] = await worker(item, index); + } + }); + await Promise.all(runners); + return results; +} + +async function main() { + const started = Date.now(); + const options = parseArguments(process.argv.slice(2)); + + /** @type {import("./differential/corpus.mjs").DifferentialCase[]} */ + const allCases = [ + ...deterministicCases(REPO_ROOT), + ...generateRandomCases(options.seed, options.count).map((entry) => ({ + id: entry.id, + category: entry.category, + bytes: Buffer.from(entry.text, "utf8"), + expect: /** @type {"oracle"} */ ("oracle"), + note: "seeded randomized input", + })), + ]; + + const selfCheck = runSelfChecks(allCases, options.count); + console.log( + `self-checks passed (${selfCheck.assertions} assertion groups, seed ${options.seed})`, + ); + if (options.selfCheckOnly) { + return; + } + + const selected = selectCases(allCases, options); + + /** @type {Map} */ + const categoryCounts = new Map(); + for (const entry of selected) { + categoryCounts.set( + entry.category, + (categoryCounts.get(entry.category) ?? 0) + 1, + ); + } + const summary = [...categoryCounts.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, total]) => `${name}=${total}`) + .join(" "); + + if (options.list) { + for (const entry of selected) { + console.log(`${entry.category}\t${entry.id}\t${entry.note ?? ""}`); + } + console.log(`\n${selected.length} cases (${summary})`); + return; + } + + if (selected.length === 0) { + throw new Error("no cases matched the requested filters"); + } + + const workspace = mkdtempSync(join(tmpdir(), "worldcut-differential-")); + let failures = 0; + try { + const runners = await prepareRunners({ + repoRoot: REPO_ROOT, + workspace, + timeoutMs: options.timeoutMs, + log: (line) => console.log(` ${line}`), + }); + console.log( + `ports: ${runners.map((runner) => `${runner.label} (${runner.description})`).join(", ")}`, + ); + console.log( + `running ${selected.length} cases (${summary}) with ${options.jobs} parallel workers`, + ); + + /** @type {Map} */ + const digests = new Map(); + /** @type {Map} */ + const verdicts = new Map(); + /** @type {Map} */ + const planStatuses = new Map(); + /** @type {Map} */ + const randomVerdicts = new Map(); + let randomSuccesses = 0; + let randomTotal = 0; + /** @type {string[]} */ + const report = []; + + const outcomes = await mapWithLimit( + selected, + options.jobs, + async (testCase, index) => { + const inputPath = join(workspace, `case-${index}.json`); + writeFileSync(inputPath, testCase.bytes); + try { + /** @type {Map} */ + const perPort = new Map(); + const results = await Promise.all( + runners.map(async (runner) => ({ + id: runner.id, + outcome: await runPort(runner, inputPath), + })), + ); + for (const entry of results) { + perPort.set(entry.id, entry.outcome); + } + return { testCase, perPort }; + } finally { + rmSync(inputPath, { force: true }); + } + }, + ); + + for (const { testCase, perPort } of outcomes) { + const problems = compareCase(testCase, runners, perPort); + const oracle = perPort.get("typescript"); + if (testCase.category === "random") { + randomTotal += 1; + if (oracle?.kind === "result") { + randomSuccesses += 1; + } + } + if (oracle?.kind === "result") { + const oracleResult = /** @type {Record} */ ( + oracle.result + ); + const digest = oracleResult["verificationRecordDigest"]; + if (typeof digest === "string") { + digests.set(testCase.id, digest); + } + const verdict = oracleResult["verdict"]; + if (typeof verdict === "string") { + verdicts.set(verdict, (verdicts.get(verdict) ?? 0) + 1); + if (testCase.category === "random") { + randomVerdicts.set( + verdict, + (randomVerdicts.get(verdict) ?? 0) + 1, + ); + } + } + const planStatus = /** @type {{ status?: unknown } | undefined} */ ( + oracleResult["acquisitionPlan"] + )?.status; + if (typeof planStatus === "string") { + planStatuses.set( + planStatus, + (planStatuses.get(planStatus) ?? 0) + 1, + ); + } + } + if (problems.length === 0) { + continue; + } + failures += 1; + if (failures <= options.maxFailures) { + report.push( + [ + `\nFAIL ${testCase.id} [${testCase.category}]`, + testCase.note === undefined ? null : ` note: ${testCase.note}`, + ` seed: ${options.seed}, count: ${options.count}`, + ` reproduce: npm run differential -- --seed ${options.seed} --count ${options.count} --only ${testCase.id}`, + describeInput(testCase.bytes), + ...problems.map((problem) => ` ${problem}`), + ] + .filter((line) => line !== null) + .join("\n"), + ); + } + } + + for (const testCase of selected) { + if (testCase.sameDigestAs === undefined) { + continue; + } + const own = digests.get(testCase.id); + const twin = digests.get(testCase.sameDigestAs); + if (own === undefined || twin === undefined) { + failures += 1; + report.push( + `\nFAIL ${testCase.id} [${testCase.category}]\n digest-equivalence twin ${testCase.sameDigestAs} did not produce a comparable result`, + ); + continue; + } + if (own !== twin) { + failures += 1; + report.push( + `\nFAIL ${testCase.id} [${testCase.category}]\n digest ${own} differs from ${testCase.sameDigestAs} digest ${twin}, but the two documents are canonically identical`, + ); + } + } + + if ( + randomTotal > 0 && + randomSuccesses < Math.ceil(randomTotal * MIN_RANDOM_SUCCESS_RATIO) + ) { + failures += 1; + report.push( + `\nFAIL generator quality\n only ${randomSuccesses}/${randomTotal} randomized cases produced a verification result; the generator must exercise the success path`, + ); + } + + if (randomTotal >= 50) { + for (const verdict of [ + "CONTRACT_SATISFIED", + "CONTRACT_VIOLATED", + "INSUFFICIENT_EVIDENCE", + ]) { + if ((randomVerdicts.get(verdict) ?? 0) === 0) { + failures += 1; + report.push( + `\nFAIL generator coverage\n no randomized case reached ${verdict}; the generator no longer exercises every verdict`, + ); + } + } + } + + for (const entry of report) { + console.error(entry); + } + if (failures > options.maxFailures) { + console.error( + `\n… ${failures - options.maxFailures} further failing cases were not printed`, + ); + } + + const elapsed = ((Date.now() - started) / 1000).toFixed(1); + if (failures === 0) { + console.log( + `\nOK ${selected.length} cases agreed across ${runners.length} implementations in ${elapsed}s`, + ); + console.log(` verdicts: ${describeCounts(verdicts)}`); + console.log(` acquisition plans: ${describeCounts(planStatuses)}`); + if (randomTotal > 0) { + console.log( + ` ${randomSuccesses}/${randomTotal} randomized cases produced a verification result (${describeCounts(randomVerdicts)})`, + ); + } + return; + } + console.error( + `\nFAILED ${failures}/${selected.length} cases diverged (${elapsed}s)`, + ); + process.exitCode = 1; + } finally { + rmSync(workspace, { recursive: true, force: true }); + } +} + +main().catch((error) => { + console.error( + `differential harness error: ${error instanceof Error ? error.message : String(error)}`, + ); + if (error instanceof Error && error.stack !== undefined) { + console.error(error.stack); + } + process.exitCode = 1; +}); diff --git a/scripts/differential/compare.mjs b/scripts/differential/compare.mjs new file mode 100644 index 0000000..f097b66 --- /dev/null +++ b/scripts/differential/compare.mjs @@ -0,0 +1,177 @@ +/** + * Outcome normalization and structural comparison for the differential + * harness. + * + * The harness compares *parsed* verification results, so indentation, member + * order, line endings, `\u` escaping, and number spelling are all irrelevant by + * construction. Everything else is treated as a semantic difference. + */ + +/** Matches the `verificationRecordDigest` shape required by `spec/0.1`. */ +export const DIGEST_PATTERN = /^[0-9a-f]{64}$/; + +/** + * The conformance suite allows a port to reject malformed transport bytes + * either in its JSON parser or in protocol validation. This maps a port's + * stable error code onto the outcome names used by + * `conformance/0.1/raw-vectors.json`. + * + * @param {string} code + * @returns {string | null} + */ +export function toRawOutcome(code) { + if (code === "WORLDCUT_INVALID_INPUT") { + return code; + } + if (code === "WORLDCUT_INVALID_JSON") { + return "PARSE_ERROR"; + } + return null; +} + +/** + * @param {unknown} value + * @returns {string} + */ +function kindOf(value) { + if (value === null) { + return "null"; + } + if (Array.isArray(value)) { + return "array"; + } + return typeof value; +} + +/** + * @param {string} segment + * @returns {string} + */ +function pointerSegment(segment) { + return segment.replaceAll("~", "~0").replaceAll("/", "~1"); +} + +/** + * Collects the semantic differences between two parsed JSON documents. + * + * Numbers are compared with `===`, so `-0` and `0` are equal. That matches + * `worldcut-json-v1`, which serializes negative zero as `0`, meaning the two + * spellings can never produce different digests. + * + * @param {unknown} expected The TypeScript oracle value. + * @param {unknown} actual The port value. + * @param {{ limit?: number }} [options] + * @returns {Array<{ path: string, expected: unknown, actual: unknown, reason: string }>} + */ +export function diffJson(expected, actual, options = {}) { + const limit = options.limit ?? 20; + /** @type {Array<{ path: string, expected: unknown, actual: unknown, reason: string }>} */ + const differences = []; + + /** + * @param {unknown} left + * @param {unknown} right + * @param {string} path + */ + const walk = (left, right, path) => { + if (differences.length >= limit) { + return; + } + const leftKind = kindOf(left); + const rightKind = kindOf(right); + if (leftKind !== rightKind) { + differences.push({ + path, + expected: left, + actual: right, + reason: `type ${leftKind} vs ${rightKind}`, + }); + return; + } + if (leftKind === "array") { + const leftItems = /** @type {unknown[]} */ (left); + const rightItems = /** @type {unknown[]} */ (right); + if (leftItems.length !== rightItems.length) { + differences.push({ + path, + expected: leftItems.length, + actual: rightItems.length, + reason: "array length", + }); + return; + } + for (let index = 0; index < leftItems.length; index += 1) { + walk(leftItems[index], rightItems[index], `${path}/${index}`); + } + return; + } + if (leftKind === "object") { + const leftRecord = /** @type {Record} */ (left); + const rightRecord = /** @type {Record} */ (right); + const keys = [ + ...new Set([...Object.keys(leftRecord), ...Object.keys(rightRecord)]), + ].sort(); + for (const key of keys) { + const hasLeft = Object.hasOwn(leftRecord, key); + const hasRight = Object.hasOwn(rightRecord, key); + if (!hasLeft || !hasRight) { + differences.push({ + path: `${path}/${pointerSegment(key)}`, + expected: hasLeft ? leftRecord[key] : "", + actual: hasRight ? rightRecord[key] : "", + reason: hasLeft ? "member missing in port" : "member absent in oracle", + }); + if (differences.length >= limit) { + return; + } + continue; + } + walk(leftRecord[key], rightRecord[key], `${path}/${pointerSegment(key)}`); + if (differences.length >= limit) { + return; + } + } + return; + } + if (left !== right) { + differences.push({ + path, + expected: left, + actual: right, + reason: "value", + }); + } + }; + + walk(expected, actual, ""); + return differences; +} + +/** + * @param {unknown} expected + * @param {unknown} actual + * @returns {boolean} + */ +export function jsonEquals(expected, actual) { + return diffJson(expected, actual, { limit: 1 }).length === 0; +} + +/** + * Renders a value for a failure report without dumping an entire document. + * + * @param {unknown} value + * @param {number} [maxLength] + * @returns {string} + */ +export function preview(value, maxLength = 160) { + let text; + try { + text = JSON.stringify(value); + } catch { + text = String(value); + } + if (text === undefined) { + return "undefined"; + } + return text.length > maxLength ? `${text.slice(0, maxLength)}…` : text; +} diff --git a/scripts/differential/corpus.mjs b/scripts/differential/corpus.mjs new file mode 100644 index 0000000..d021851 --- /dev/null +++ b/scripts/differential/corpus.mjs @@ -0,0 +1,1278 @@ +/** + * The deterministic half of the differential corpus. + * + * Every committed conformance vector, every published example, and a set of + * handcrafted edge cases are turned into transport bytes here. Cases that + * depend on a number's lexical form, on duplicate member names, or on raw + * encoding are authored as text or bytes so that no JavaScript value ever + * normalizes them before a CLI sees them. + */ + +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { members, raw, toJsonText } from "./json-text.mjs"; + +const DECISION_TIME = "2026-09-02T18:00:00.000Z"; +const OBSERVED_AT = "2026-09-02T17:59:00.000Z"; + +/** + * @typedef {object} DifferentialCase + * @property {string} id + * @property {string} category + * @property {Buffer} bytes Exact transport bytes handed to every CLI. + * @property {"result" | "oracle" | "code" | "transport"} expect + * @property {string} [code] Required stable error code when `expect` is `code`. + * @property {string[]} [allowed] Accepted outcomes when `expect` is `transport`. + * @property {unknown} [expectedResult] Committed golden result, when one exists. + * @property {string} [sameDigestAs] Another case id that must produce the same digest. + * @property {string} [note] Why the case exists. + */ + +/** + * @param {unknown} value + * @param {number} [indent] + * @returns {Buffer} + */ +function encode(value, indent = 2) { + return Buffer.from(toJsonText(value, { indent }), "utf8"); +} + +/** + * @param {string} key + * @returns {Record} + */ +function resource(key) { + return { + provider: "github", + account: "acme", + kind: "branch_head", + key, + }; +} + +/** + * @param {object} options + * @param {string} options.role + * @param {unknown} options.value + * @param {number} [options.cost] + * @param {Record} [options.witness] + * @param {string} [options.observedAt] + * @param {Record} [options.resource] + * @returns {Record} + */ +function observation(options) { + return { + id: `obs-${options.role}`, + role: options.role, + resource: options.resource ?? resource(options.role), + value: options.value, + observedAt: options.observedAt ?? OBSERVED_AT, + acquisitionCost: options.cost ?? 1, + witness: options.witness ?? { provenance: "provider_asserted" }, + }; +} + +/** + * @param {object} options + * @param {Array>} options.requirements + * @param {Array>} options.observations + * @param {string} [options.id] + * @param {string} [options.decisionTime] + * @returns {Record} + */ +function input(options) { + return { + protocolVersion: "0.1", + contract: { + id: options.id ?? "differential-edge", + version: "1", + decisionTime: options.decisionTime ?? DECISION_TIME, + assumptions: { + clockModel: "trusted_normalized", + intervalModel: "half_open", + metadataModel: "honest_but_possibly_incomplete", + }, + requirements: options.requirements, + }, + observations: options.observations, + }; +} + +/** + * @param {string} id + * @param {string} role + * @param {string[]} path + * @param {unknown} expected + * @param {boolean} [required] + * @returns {Record} + */ +function valueEquals(id, role, path, expected, required) { + /** @type {Record} */ + const requirement = { + id, + type: "value_equals", + description: `Value at ${path.join("/")} for ${role}`, + role, + path, + expected, + }; + if (required === false) { + requirement["required"] = false; + } + return requirement; +} + +/** + * @param {string} id + * @param {string} dependentRole + * @param {string} targetRole + * @param {string} dependencyName + * @returns {Record} + */ +function dependency(id, dependentRole, targetRole, dependencyName) { + return { + id, + type: "dependency", + description: `${dependentRole} depends on ${targetRole}`, + dependentRole, + targetRole, + dependencyName, + }; +} + +/** + * @param {string} id + * @param {string[]} roles + * @param {string} from + * @param {string | null} until + * @returns {Record} + */ +function commonValidTime(id, roles, from, until) { + return { + id, + type: "common_valid_time", + description: `${roles.join(" and ")} share a valid time`, + roles, + within: { from, until }, + }; +} + +/** + * Handcrafted edge cases that pin behaviour the golden vectors do not cover. + * + * @returns {DifferentialCase[]} + */ +function edgeCases() { + /** @type {DifferentialCase[]} */ + const cases = []; + + /** + * @param {string} id + * @param {unknown} document + * @param {Partial} [extra] + */ + const add = (id, document, extra = {}) => { + cases.push({ + id: `edge/${id}`, + category: "edge", + bytes: Buffer.isBuffer(document) ? document : encode(document), + expect: "result", + ...extra, + }); + }; + + // --- Number lexical forms ------------------------------------------------- + + add( + "number-underflow-positive", + input({ + requirements: [valueEquals("tiny", "a", ["tiny"], raw("0"))], + observations: [ + observation({ + role: "a", + value: members([ + ["tiny", raw("1e-400")], + ["huge", raw("1.7976931348623157e308")], + ]), + }), + ], + }), + { + note: "finite IEEE-754 underflow must parse as 0 in every port", + }, + ); + + add( + "number-underflow-negative", + input({ + requirements: [valueEquals("tiny", "a", ["tiny"], raw("-0"))], + observations: [ + observation({ + role: "a", + value: members([["tiny", raw("-1e-400")]]), + }), + ], + }), + { + note: "negative underflow must parse as negative zero and canonicalize to 0", + }, + ); + + add( + "number-underflow-nested", + input({ + requirements: [valueEquals("tiny", "a", ["deep", "0", "x"], raw("0"))], + observations: [ + observation({ + role: "a", + value: members([ + ["deep", [members([["x", raw("2e-400")]])]], + ["also", [raw("-1e-999"), raw("1e-320")]], + ]), + }), + ], + }), + { note: "underflow inside nested containers" }, + ); + + add( + "number-negative-zero", + input({ + requirements: [valueEquals("zero", "a", ["z"], raw("0"))], + observations: [ + observation({ role: "a", value: members([["z", raw("-0")]]) }), + ], + }), + { note: "worldcut-json-v1 serializes negative zero as 0" }, + ); + + add( + "number-lexical-forms", + input({ + requirements: [valueEquals("hundred", "a", ["b"], raw("100"))], + observations: [ + observation({ + role: "a", + value: members([ + ["a", raw("1.0")], + ["b", raw("1E+2")], + ["c", raw("0.1")], + ["d", raw("1e21")], + ["e", raw("-1.5e-7")], + ["f", raw("3.0e0")], + ]), + }), + ], + }), + { note: "exponent and trailing-zero spellings must be value-equal" }, + ); + + add( + "number-boundaries", + input({ + requirements: [valueEquals("max", "a", ["max"], raw("1.7976931348623157e308"))], + observations: [ + observation({ + role: "a", + value: members([ + ["max", raw("1.7976931348623157e308")], + ["minNormal", raw("2.2250738585072014e-308")], + ["minSubnormal", raw("5e-324")], + ["maxSafe", raw("9007199254740991")], + ["minSafe", raw("-9007199254740991")], + ["beyondSafe", raw("9007199254740993")], + ]), + }), + ], + }), + { note: "IEEE-754 boundary doubles and integer precision loss" }, + ); + + // --- Unicode and UTF-16 ordering ----------------------------------------- + + add( + "unicode-key-ordering", + input({ + requirements: [ + valueEquals("k1", "a", ["Ｚ"], "fullwidth"), + valueEquals("k2", "a", ["𝄞"], "astral"), + valueEquals("k3", "a", ["ä"], "latin"), + ], + observations: [ + observation({ + role: "a", + value: members([ + ["𝄞", "astral"], + ["z", "lower"], + ["Ｚ", "fullwidth"], + ["Z", "upper"], + ["ä", "latin"], + ["", "empty-key"], + ]), + }), + ], + }), + { note: "canonical member ordering by raw UTF-16 code units" }, + ); + + add( + "unicode-key-ordering-reordered", + input({ + requirements: [ + valueEquals("k3", "a", ["ä"], "latin"), + valueEquals("k2", "a", ["𝄞"], "astral"), + valueEquals("k1", "a", ["Ｚ"], "fullwidth"), + ], + observations: [ + observation({ + role: "a", + value: members([ + ["", "empty-key"], + ["ä", "latin"], + ["Z", "upper"], + ["Ｚ", "fullwidth"], + ["z", "lower"], + ["𝄞", "astral"], + ]), + }), + ], + }), + { + sameDigestAs: "edge/unicode-key-ordering", + note: "member and requirement order must not change the digest", + }, + ); + + add( + "unicode-strings", + input({ + requirements: [valueEquals("s", "a", ["text"], "line\nbreak\ttab \"q\" \\ Ω≈ç 𝄞")], + observations: [ + observation({ + role: "a", + value: members([ + ["text", "line\nbreak\ttab \"q\" \\ Ω≈ç 𝄞"], + ["combining", "e\u0301 vs \u00e9"], + ["control", "\u0001\u001f"], + ]), + }), + ], + }), + { note: "escaping, combining marks, and control characters" }, + ); + + add( + "unicode-escaped-input", + Buffer.from( + toJsonText( + input({ + requirements: [valueEquals("s", "a", ["text"], "Ω 𝄞")], + observations: [ + observation({ role: "a", value: members([["text", "Ω 𝄞"]]) }), + ], + }), + { indent: 2 }, + ).replaceAll("Ω", "\\u03a9").replaceAll("𝄞", "\\ud834\\udd1e"), + "utf8", + ), + { + sameDigestAs: "edge/unicode-escaped-literal", + note: "\\u escapes and literal UTF-8 must be the same document", + }, + ); + + add( + "unicode-escaped-literal", + input({ + requirements: [valueEquals("s", "a", ["text"], "Ω 𝄞")], + observations: [ + observation({ role: "a", value: members([["text", "Ω 𝄞"]]) }), + ], + }), + { note: "literal UTF-8 counterpart of the escaped case" }, + ); + + // --- value_equals path handling ------------------------------------------ + + add( + "path-whitespace-members", + input({ + requirements: [ + valueEquals("space", "a", [" "], "single-space"), + valueEquals("inner", "a", ["a b"], "inner-space"), + ], + observations: [ + observation({ + role: "a", + value: members([ + [" ", "single-space"], + ["a b", "inner-space"], + ]), + }), + ], + }), + { note: "whitespace member names remain addressable" }, + ); + + add( + "path-array-index", + input({ + requirements: [ + valueEquals("first", "a", ["items", "0"], "alpha"), + valueEquals("nested", "a", ["items", "2", "k"], "deep"), + valueEquals("length", "a", ["items", "length"], 3, false), + valueEquals("leadingZero", "a", ["items", "00"], "alpha", false), + ], + observations: [ + observation({ + role: "a", + value: members([ + ["items", ["alpha", "beta", members([["k", "deep"]])]], + ]), + }), + ], + }), + { note: "array indexing, and `length` / `00` are not value paths" }, + ); + + add( + "path-missing-and-null", + input({ + requirements: [ + valueEquals("missing", "a", ["absent"], "x"), + valueEquals("null", "a", ["nothing"], null, false), + valueEquals("throughNull", "a", ["nothing", "x"], "x", false), + valueEquals("empty", "a", ["object"], members([]), false), + ], + observations: [ + observation({ + role: "a", + value: members([ + ["nothing", null], + ["object", members([])], + ["array", []], + ]), + }), + ], + }), + { note: "absent paths, null values, and empty containers" }, + ); + + add( + "value-equals-structural", + input({ + requirements: [ + valueEquals( + "reordered", + "a", + ["payload"], + members([ + ["b", [1, members([["y", 2]])]], + ["a", raw("1.0")], + ]), + ), + ], + observations: [ + observation({ + role: "a", + value: members([ + [ + "payload", + members([ + ["a", raw("1")], + ["b", [raw("1e0"), members([["y", raw("2.0")]])]], + ]), + ], + ]), + }), + ], + }), + { note: "structural equality is canonical, not textual" }, + ); + + add( + "deep-nesting", + (() => { + /** @type {unknown} */ + let value = "bottom"; + /** @type {string[]} */ + const path = []; + for (let level = 0; level < 24; level += 1) { + value = members([["n", value]]); + path.unshift("n"); + } + return input({ + requirements: [valueEquals("deep", "a", path, "bottom")], + observations: [observation({ role: "a", value })], + }); + })(), + { note: "deep but legal nesting, far below the 48-level transport cap" }, + ); + + // --- dependency evaluation ----------------------------------------------- + + /** + * @param {string | undefined} version + * @param {Record} [target] + * @returns {Record} + */ + const dependencyWitness = (version, target) => { + /** @type {Record} */ + const record = { + name: "tested_head", + resource: target ?? resource("b"), + relation: "exact", + provenance: "provider_asserted", + }; + if (version !== undefined) { + record["version"] = version; + } + return { + provenance: "provider_asserted", + version: "run-1", + dependencies: [record], + }; + }; + + add( + "dependency-satisfied", + input({ + requirements: [dependency("d", "a", "b", "tested_head")], + observations: [ + observation({ + role: "a", + value: members([["status", "passed"]]), + witness: dependencyWitness("commit-B"), + cost: 4, + }), + observation({ + role: "b", + value: members([["commit", "commit-B"]]), + witness: { provenance: "provider_asserted", version: "commit-B" }, + }), + ], + }), + { note: "matching dependency version" }, + ); + + add( + "dependency-version-violated", + input({ + requirements: [dependency("d", "a", "b", "tested_head")], + observations: [ + observation({ + role: "a", + value: members([["status", "passed"]]), + witness: dependencyWitness("commit-A"), + cost: 4, + }), + observation({ + role: "b", + value: members([["commit", "commit-B"]]), + witness: { provenance: "provider_asserted", version: "commit-B" }, + }), + ], + }), + { note: "violated dependency produces two acquisition options" }, + ); + + add( + "dependency-resource-violated", + input({ + requirements: [dependency("d", "a", "b", "tested_head")], + observations: [ + observation({ + role: "a", + value: members([["status", "passed"]]), + witness: dependencyWitness("commit-B", resource("other")), + cost: 4, + }), + observation({ + role: "b", + value: members([["commit", "commit-B"]]), + witness: { provenance: "provider_asserted", version: "commit-B" }, + }), + ], + }), + { note: "dependency bound to a different resource" }, + ); + + add( + "dependency-unknown-version", + input({ + requirements: [dependency("d", "a", "b", "tested_head")], + observations: [ + observation({ + role: "a", + value: members([["status", "passed"]]), + witness: dependencyWitness(undefined), + cost: 4, + }), + observation({ + role: "b", + value: members([["commit", "commit-B"]]), + witness: { provenance: "provider_asserted" }, + }), + ], + }), + { note: "both versions missing produces a metadata fetch plan" }, + ); + + add( + "dependency-unknown-name", + input({ + requirements: [dependency("d", "a", "b", "not_declared")], + observations: [ + observation({ + role: "a", + value: members([["status", "passed"]]), + witness: dependencyWitness("commit-B"), + cost: 9, + }), + observation({ + role: "b", + value: members([["commit", "commit-B"]]), + witness: { provenance: "provider_asserted", version: "commit-B" }, + }), + ], + }), + { note: "declared dependency name is absent from the witness" }, + ); + + add( + "dependency-missing-roles", + input({ + requirements: [dependency("d", "absent", "b", "tested_head")], + observations: [ + observation({ + role: "b", + value: members([["commit", "commit-B"]]), + witness: { provenance: "provider_asserted", version: "commit-B" }, + }), + anchorObservation(), + ], + }), + { note: "unbound roles fail closed with an acquisition action" }, + ); + + // --- temporal evaluation -------------------------------------------------- + + add( + "temporal-overlap", + input({ + requirements: [ + commonValidTime( + "t", + ["a", "b"], + "2026-09-02T17:55:00.000Z", + "2026-09-02T18:00:00.001Z", + ), + ], + observations: [ + observation({ + role: "a", + value: members([["ok", true]]), + witness: { + provenance: "provider_asserted", + validity: { + from: "2026-09-02T17:50:00.000Z", + until: "2026-09-02T18:10:00.000Z", + }, + }, + }), + observation({ + role: "b", + value: members([["ok", true]]), + witness: { + provenance: "provider_asserted", + validity: { + from: "2026-09-02T17:58:00.000Z", + until: null, + }, + }, + }), + ], + }), + { note: "open-ended and closed validity overlap" }, + ); + + add( + "temporal-gap", + input({ + requirements: [ + commonValidTime( + "t", + ["a", "b"], + "2026-09-02T17:00:00.000Z", + "2026-09-02T18:00:00.000Z", + ), + ], + observations: [ + observation({ + role: "a", + value: members([["ok", true]]), + witness: { + provenance: "provider_asserted", + validity: { + from: "2026-09-02T17:00:00.000Z", + until: "2026-09-02T17:30:00.000Z", + }, + }, + }), + observation({ + role: "b", + value: members([["ok", true]]), + witness: { + provenance: "provider_asserted", + validity: { + from: "2026-09-02T17:30:00.000Z", + until: "2026-09-02T17:59:00.000Z", + }, + }, + }), + ], + }), + { note: "half-open intervals that touch but never overlap" }, + ); + + add( + "temporal-missing-validity", + input({ + requirements: [ + commonValidTime( + "t", + ["a", "b", "c"], + "2026-09-02T17:00:00.000Z", + null, + ), + ], + observations: [ + observation({ + role: "a", + value: members([["ok", true]]), + cost: 12, + }), + observation({ + role: "b", + value: members([["ok", true]]), + cost: 40, + witness: { + provenance: "provider_asserted", + validity: { + from: "2026-09-02T17:00:00.000Z", + until: null, + }, + }, + }), + observation({ role: "c", value: members([["ok", true]]), cost: 3 }), + ], + }), + { note: "missing validity metadata drives a fetch plan" }, + ); + + // --- planning, dedup, and aggregation ------------------------------------ + + add( + "plan-deduplicates-actions", + input({ + requirements: [ + valueEquals("v1", "a", ["missing1"], "x"), + valueEquals("v2", "a", ["missing2"], "y"), + valueEquals("v3", "b", ["missing3"], "z"), + ], + observations: [ + observation({ + role: "a", + value: members([["status", "passed"]]), + cost: 7, + }), + observation({ + role: "b", + value: members([["status", "passed"]]), + cost: 11, + }), + ], + }), + { note: "identical refresh actions must be counted once" }, + ); + + add( + "aggregation-advisory-only", + input({ + requirements: [ + valueEquals("required-ok", "a", ["status"], "passed"), + valueEquals("advisory-bad", "a", ["status"], "failed", false), + valueEquals("advisory-unknown", "a", ["absent"], "x", false), + ], + observations: [ + observation({ role: "a", value: members([["status", "passed"]]) }), + ], + }), + { note: "advisory failures must not change the verdict" }, + ); + + add( + "aggregation-violation-dominates", + input({ + requirements: [ + valueEquals("violated", "a", ["status"], "failed"), + valueEquals("unknown", "a", ["absent"], "x"), + ], + observations: [ + observation({ role: "a", value: members([["status", "passed"]]) }), + ], + }), + { note: "a violation outranks an unknown" }, + ); + + add( + "cost-boundaries", + input({ + requirements: [ + valueEquals("v1", "a", ["absent"], "x"), + valueEquals("v2", "b", ["absent"], "x"), + ], + observations: [ + observation({ + role: "a", + value: members([["status", "passed"]]), + cost: 1000000000, + }), + observation({ + role: "b", + value: members([["status", "passed"]]), + cost: 0, + }), + ], + }), + { note: "maximum and zero acquisition costs" }, + ); + + // --- ordering independence ------------------------------------------------ + + const orderingRequirements = [ + valueEquals("z-last", "a", ["status"], "passed"), + dependency("m-mid", "a", "b", "tested_head"), + commonValidTime( + "a-first", + ["a", "b"], + "2026-09-02T17:00:00.000Z", + "2026-09-02T18:00:00.001Z", + ), + ]; + const orderingObservations = [ + observation({ + role: "a", + value: members([["status", "passed"]]), + cost: 5, + witness: { + provenance: "provider_asserted", + version: "run-1", + validity: { + from: "2026-09-02T17:30:00.000Z", + until: null, + }, + dependencies: [ + { + name: "tested_head", + resource: resource("b"), + relation: "exact", + version: "commit-B", + provenance: "provider_asserted", + }, + ], + }, + }), + observation({ + role: "b", + value: members([["commit", "commit-B"]]), + cost: 2, + witness: { + provenance: "provider_asserted", + version: "commit-B", + validity: { + from: "2026-09-02T17:40:00.000Z", + until: "2026-09-02T18:40:00.000Z", + }, + }, + }), + ]; + + add( + "ordering-declared", + input({ + requirements: orderingRequirements, + observations: orderingObservations, + }), + { note: "baseline ordering for the reordered twin" }, + ); + + add( + "ordering-reversed", + input({ + requirements: [...orderingRequirements].reverse(), + observations: [...orderingObservations].reverse(), + }), + { + sameDigestAs: "edge/ordering-declared", + note: "input array order must not affect the verification record", + }, + ); + + return cases; +} + +/** + * A second observation used to keep an unbound-role case otherwise valid. + * + * @returns {Record} + */ +function anchorObservation() { + return observation({ + role: "anchor", + value: members([["status", "passed"]]), + cost: 3, + }); +} + +/** + * Malformed or ambiguous transport bytes. + * + * `spec/0.1/CONFORMANCE.md` lets an implementation reject these either in its + * JSON parser or in protocol validation, so the harness asserts that every port + * fails with one of the accepted outcomes rather than with identical prose. + * + * @param {string} repoRoot + * @returns {DifferentialCase[]} + */ +function transportCases(repoRoot) { + const parseOrInvalid = ["PARSE_ERROR", "WORLDCUT_INVALID_INPUT"]; + + const valid = toJsonText( + input({ + requirements: [valueEquals("v", "a", ["status"], "passed")], + observations: [ + observation({ role: "a", value: members([["status", "passed"]]) }), + ], + }), + { indent: 2 }, + ); + + /** @type {DifferentialCase[]} */ + const cases = []; + + /** + * @param {string} id + * @param {Buffer} bytes + * @param {string} note + * @param {string[]} [allowed] + */ + const add = (id, bytes, note, allowed = parseOrInvalid) => { + cases.push({ + id: `transport/${id}`, + category: "transport", + bytes, + expect: "transport", + allowed, + note, + }); + }; + + add("empty", Buffer.alloc(0), "an empty file is not a JSON document"); + add( + "whitespace-only", + Buffer.from(" \n\t\r\n", "utf8"), + "whitespace without a value", + ); + add( + "truncated-object", + Buffer.from(valid.slice(0, Math.floor(valid.length / 2)), "utf8"), + "a document cut in half", + ); + add( + "trailing-value", + Buffer.from(`${valid}\n{}\n`, "utf8"), + "a second JSON value after the input", + ); + add( + "trailing-comma", + Buffer.from(`${valid.slice(0, -1)},}`, "utf8"), + "a trailing comma is not JSON", + ); + add( + "byte-order-mark", + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(valid, "utf8")]), + "a UTF-8 byte-order mark before the document", + ); + add( + "invalid-utf8", + Buffer.concat([ + Buffer.from(valid.slice(0, valid.indexOf("passed")), "utf8"), + Buffer.from([0xff, 0xfe]), + Buffer.from(valid.slice(valid.indexOf("passed")), "utf8"), + ]), + "bytes that are not valid UTF-8", + ); + add( + "raw-control-character", + Buffer.from(valid.replace('"passed"', '"pas\u0001sed"'), "utf8"), + "an unescaped control character inside a string", + ); + add( + "nul-byte", + Buffer.concat([Buffer.from(valid, "utf8"), Buffer.from([0x00])]), + "a NUL byte after the document", + ); + add( + "lone-high-surrogate", + Buffer.from(valid.replace('"passed"', '"\\ud834"'), "utf8"), + "an unpaired high surrogate escape", + ); + add( + "lone-low-surrogate", + Buffer.from(valid.replace('"passed"', '"\\udd1e"'), "utf8"), + "an unpaired low surrogate escape", + ); + add( + "nan-literal", + Buffer.from(valid.replace('"acquisitionCost": 1', '"acquisitionCost": NaN'), "utf8"), + "NaN is not a JSON number", + ); + add( + "infinity-literal", + Buffer.from( + valid.replace('"acquisitionCost": 1', '"acquisitionCost": Infinity'), + "utf8", + ), + "Infinity is not a JSON number", + ); + add( + "number-overflow", + Buffer.from( + toJsonText( + input({ + requirements: [valueEquals("v", "a", ["big"], raw("1"))], + observations: [ + observation({ role: "a", value: members([["big", raw("1e400")]]) }), + ], + }), + { indent: 2 }, + ), + "utf8", + ), + "a number that overflows to infinity must be rejected", + ["WORLDCUT_INVALID_INPUT", "PARSE_ERROR"], + ); + add( + "number-overflow-negative", + Buffer.from( + toJsonText( + input({ + requirements: [valueEquals("v", "a", ["big"], raw("1"))], + observations: [ + observation({ role: "a", value: members([["big", raw("-1e400")]]) }), + ], + }), + { indent: 2 }, + ), + "utf8", + ), + "negative overflow must be rejected", + ["WORLDCUT_INVALID_INPUT", "PARSE_ERROR"], + ); + add( + "leading-zero-number", + Buffer.from(valid.replace('"acquisitionCost": 1', '"acquisitionCost": 01'), "utf8"), + "leading zeros are not JSON numbers", + ); + add( + "hex-number", + Buffer.from(valid.replace('"acquisitionCost": 1', '"acquisitionCost": 0x1'), "utf8"), + "hexadecimal is not a JSON number", + ); + add( + "single-quoted-string", + Buffer.from(valid.replace('"passed"', "'passed'"), "utf8"), + "single quotes are not JSON strings", + ); + add( + "top-level-array", + Buffer.from("[]\n", "utf8"), + "the protocol input must be an object", + ); + add( + "top-level-string", + Buffer.from('"0.1"\n', "utf8"), + "a bare string is not a verification input", + ); + + const raws = JSON.parse( + readFileSync(join(repoRoot, "conformance", "0.1", "raw-vectors.json"), "utf8"), + ); + for (const vector of raws.cases) { + cases.push({ + id: `raw/${vector.name}`, + category: "raw", + bytes: readFileSync( + join(repoRoot, "conformance", "0.1", ...vector.file.split("/")), + ), + expect: "transport", + allowed: vector.acceptedOutcomes, + note: "committed raw conformance vector", + }); + } + + return cases; +} + +/** + * Verifies the committed conformance files before they become differential + * inputs. This prevents a stale or locally corrupted corpus from weakening the + * comparison while still looking like a successful run. + * + * @param {string} repoRoot + * @returns {void} + */ +function verifyConformanceCorpus(repoRoot) { + const root = join(repoRoot, "conformance", "0.1"); + const manifest = JSON.parse(readFileSync(join(root, "manifest.json"), "utf8")); + for (const [name, metadata] of Object.entries(manifest.files)) { + const bytes = readFileSync(join(root, ...name.split("/"))); + const digest = createHash("sha256").update(bytes).digest("hex"); + if (digest !== metadata.sha256) { + throw new Error( + `conformance file ${name} has SHA-256 ${digest}, expected ${metadata.sha256}`, + ); + } + if (metadata.bytes !== undefined && bytes.length !== metadata.bytes) { + throw new Error( + `conformance file ${name} has ${bytes.length} bytes, expected ${metadata.bytes}`, + ); + } + if (metadata.cases !== undefined) { + const parsed = JSON.parse(bytes.toString("utf8")); + if (!Array.isArray(parsed.cases) || parsed.cases.length !== metadata.cases) { + throw new Error( + `conformance file ${name} has ${String(parsed.cases?.length)} cases, expected ${metadata.cases}`, + ); + } + } + } +} + +/** + * Applies user filters while keeping canonical-equivalence pairs together. + * Selecting either side of a `sameDigestAs` relationship automatically pulls + * in the other side so a focused reproduction cannot silently skip the digest + * assertion it is meant to exercise. + * + * @param {DifferentialCase[]} cases + * @param {{ category: string | null, only: string | null }} filters + * @returns {DifferentialCase[]} + */ +export function selectCases(cases, filters) { + const selectedIds = new Set( + cases + .filter((entry) => { + if (filters.category !== null && entry.category !== filters.category) { + return false; + } + return filters.only === null || entry.id.includes(filters.only); + }) + .map((entry) => entry.id), + ); + + let changed = true; + while (changed) { + changed = false; + for (const entry of cases) { + if ( + selectedIds.has(entry.id) && + entry.sameDigestAs !== undefined && + !selectedIds.has(entry.sameDigestAs) + ) { + selectedIds.add(entry.sameDigestAs); + changed = true; + } + if ( + entry.sameDigestAs !== undefined && + selectedIds.has(entry.sameDigestAs) && + !selectedIds.has(entry.id) + ) { + selectedIds.add(entry.id); + changed = true; + } + } + } + + return cases.filter((entry) => selectedIds.has(entry.id)); +} + +/** + * Assembles every deterministic case. + * + * @param {string} repoRoot + * @returns {DifferentialCase[]} + */ +export function deterministicCases(repoRoot) { + verifyConformanceCorpus(repoRoot); + + /** @type {DifferentialCase[]} */ + const cases = []; + + const verification = JSON.parse( + readFileSync( + join(repoRoot, "conformance", "0.1", "verification-vectors.json"), + "utf8", + ), + ); + for (const vector of verification.cases) { + cases.push({ + id: `golden/${vector.name}`, + category: "golden", + bytes: Buffer.from(JSON.stringify(vector.input, null, 2), "utf8"), + expect: "result", + expectedResult: vector.expected, + note: "committed verification vector", + }); + } + + const invalid = JSON.parse( + readFileSync( + join(repoRoot, "conformance", "0.1", "invalid-vectors.json"), + "utf8", + ), + ); + for (const vector of invalid.cases) { + cases.push({ + id: `invalid/${vector.name}`, + category: "invalid", + bytes: Buffer.from(JSON.stringify(vector.input, null, 2), "utf8"), + expect: "code", + code: vector.expectedErrorCode, + note: "committed invalid vector", + }); + } + + for (const name of [ + "coherent-deployment", + "git-ci-mismatch", + "missing-evidence", + "temporal-gap", + ]) { + cases.push({ + id: `example/${name}`, + category: "example", + bytes: readFileSync(join(repoRoot, "examples", `${name}.json`)), + expect: "result", + note: "published example", + }); + } + + cases.push(...edgeCases()); + cases.push(...transportCases(repoRoot)); + + return cases; +} diff --git a/scripts/differential/generate.mjs b/scripts/differential/generate.mjs new file mode 100644 index 0000000..d8f98d4 --- /dev/null +++ b/scripts/differential/generate.mjs @@ -0,0 +1,530 @@ +/** + * Seeded generator for randomized but structurally valid WorldCut verification + * inputs. + * + * Every case is produced from `${seed}:${index}` alone, so the same seed and + * count always yield byte-identical transport text on every platform. Nesting + * stays far below the 48-level transport cap documented by the .NET port. + */ + +import { DeterministicRandom } from "./prng.mjs"; +import { members, raw, toJsonText } from "./json-text.mjs"; + +/** Fixed decision-time anchor so generated timestamps never depend on the clock. */ +const BASE_TIME_MS = Date.UTC(2026, 8, 2, 18, 0, 0); + +const ROLE_POOL = [ + "head", + "ci", + "approval", + "quote", + "deploy", + "déployé", + "ロール", + "role-Ω", + "rôle_β", + "Ａ", + "z", + "Z", +]; + +const PROVIDER_POOL = [ + "github", + "ci.example", + "change.example", + "pricing.example", + "registry.例", +]; + +const ACCOUNT_POOL = ["acme", "acme-ünïcode", "組織"]; + +const KIND_POOL = ["branch_head", "ci_run", "approval", "quote", "artifact"]; + +const VERSION_POOL = [ + "commit-A", + "commit-B", + "v1.0.0", + "版-1", + "sha-0f1e2d", + "Z", + "z", +]; + +const PROVENANCE_POOL = [ + "provider_asserted", + "client_observed", + "derived", + "operator_supplied", +]; + +const DEPENDENCY_NAME_POOL = ["tested_head", "source", "input", "依存", "a b"]; + +/** + * Member names chosen to exercise UTF-16 code-unit ordering during + * canonicalization: ASCII, Latin-1, full-width, and an astral pair. + */ +const KEY_POOL = [ + "status", + "commit", + "count", + "ok", + "nested", + "items", + "a b", + " ", + "Z", + "z", + "ä", + "Ｚ", + "ß", + "0", + "１", + "𝄞", +]; + +const STRING_POOL = [ + "", + "passed", + "failed", + "commit-A", + "commit-B", + "line\nbreak", + "tab\tstop", + 'quote"inside', + "back\\slash", + "Ω≈ç√∫", + "日本語テキスト", + "𝄞 clef", + "Ｚ", + "z", +]; + +/** + * Number lexemes that `JSON.stringify` would erase before a CLI ever saw them, + * including finite IEEE-754 underflow and negative zero. + */ +const RAW_NUMBER_POOL = [ + "-0", + "0", + "0.0", + "1e-400", + "-1e-400", + "1E+2", + "1.0", + "0.1", + "1e21", + "-1.5e-7", + "9007199254740991", + "-9007199254740991", + "5e-324", + "1.7976931348623157e308", + "2.2250738585072014e-308", + "3.0e0", +]; + +const NUMBER_POOL = [ + 0, 1, -1, 2, 42, 1000, 65535, 3.5, 0.25, 3.141592653589793, 1e-7, 1.5e10, + 9007199254740991, -9007199254740991, +]; + +/** + * @param {number} milliseconds + * @returns {string} + */ +function timestamp(milliseconds) { + return new Date(milliseconds).toISOString(); +} + +/** + * @param {DeterministicRandom} rng + * @returns {string} + */ +function randomString(rng) { + if (rng.chance(0.2)) { + return `s-${rng.below(1000)}`; + } + return rng.pick(STRING_POOL); +} + +/** + * @param {DeterministicRandom} rng + * @returns {unknown} + */ +function randomScalar(rng) { + const roll = rng.below(6); + if (roll === 0) { + return null; + } + if (roll === 1) { + return rng.chance(0.5); + } + if (roll === 2) { + return rng.pick(NUMBER_POOL); + } + if (roll === 3) { + return raw(rng.pick(RAW_NUMBER_POOL)); + } + return randomString(rng); +} + +/** + * @param {DeterministicRandom} rng + * @param {number} depth + * @returns {unknown} + */ +function randomValue(rng, depth) { + if (depth >= 3 || rng.chance(0.45)) { + return randomScalar(rng); + } + if (rng.chance(0.5)) { + const length = rng.below(4); + const items = []; + for (let index = 0; index < length; index += 1) { + items.push(randomValue(rng, depth + 1)); + } + return items; + } + const size = rng.below(4) + 1; + const keys = rng.sample(KEY_POOL, size); + return members(keys.map((key) => [key, randomValue(rng, depth + 1)])); +} + +/** + * Enumerates every `value_equals` path reachable inside a generated value. + * + * @param {unknown} node + * @param {string[]} prefix + * @param {Array<{ path: string[], node: unknown }>} sink + * @returns {void} + */ +function collectPaths(node, prefix, sink) { + if (prefix.length > 0) { + sink.push({ path: [...prefix], node }); + } + if (prefix.length >= 4) { + return; + } + if (Array.isArray(node)) { + node.forEach((item, index) => { + collectPaths(item, [...prefix, String(index)], sink); + }); + return; + } + if (node !== null && typeof node === "object" && "entries" in node) { + for (const [key, value] of /** @type {{ entries: [string, unknown][] }} */ ( + node + ).entries) { + if (key.length === 0) { + continue; + } + collectPaths(value, [...prefix, key], sink); + } + } +} + +/** + * @param {DeterministicRandom} rng + * @param {number} index + * @returns {Record} + */ +function randomResource(rng, index) { + return { + provider: rng.pick(PROVIDER_POOL), + account: rng.pick(ACCOUNT_POOL), + kind: rng.pick(KIND_POOL), + key: rng.chance(0.3) ? `キー/${index}` : `key-${index}-${rng.below(4)}`, + }; +} + +/** + * Builds one randomized verification input. + * + * About a third of the corpus is generated in "coherent" mode, where every + * requirement is constructed to be satisfiable. Without it the random corpus + * almost never reaches `CONTRACT_SATISFIED` or a `NOT_NEEDED` acquisition plan. + * + * @param {DeterministicRandom} rng + * @returns {{ text: string }} + */ +function buildInput(rng) { + const coherent = rng.chance(0.35); + const decisionTimeMs = BASE_TIME_MS + rng.between(0, 5_000_000); + const roles = rng.sample(ROLE_POOL, rng.between(2, 5)); + + /** + * @type {Array<{ + * role: string, + * record: Record, + * resource: Record, + * version: string | null, + * value: unknown, + * links: Array<{ name: string, targetRole: string, coherent: boolean }>, + * }>} + */ + const observations = []; + + roles.forEach((role, index) => { + const resource = randomResource(rng, index); + const value = randomValue(rng, 0); + const version = coherent || rng.chance(0.8) ? rng.pick(VERSION_POOL) : null; + /** @type {Record} */ + const witness = { provenance: rng.pick(PROVENANCE_POOL) }; + if (version !== null) { + witness["version"] = version; + } + if (coherent) { + witness["validity"] = { + from: timestamp(decisionTimeMs - 7_200_000), + until: rng.chance(0.25) + ? null + : timestamp(decisionTimeMs + 3_600_000), + }; + } else if (rng.chance(0.75)) { + const fromMs = decisionTimeMs - rng.between(0, 7_200_000); + const openEnded = rng.chance(0.2); + witness["validity"] = { + from: timestamp(fromMs), + until: openEnded + ? null + : timestamp(fromMs + rng.between(1, 5_400_000)), + }; + } + observations.push({ + role, + resource, + version, + value, + links: [], + record: { + id: `obs-${index}-${rng.below(1000)}`, + role, + resource, + value, + observedAt: timestamp(decisionTimeMs - rng.between(0, 3_600_000)), + acquisitionCost: rng.chance(0.1) + ? rng.pick([0, 1, 1_000_000_000]) + : rng.between(0, 5000), + witness, + }, + }); + }); + + // Dependency witnesses reference sibling observations so the differential + // corpus covers satisfied, violated, and unknown dependency evaluation. + for (const observation of observations) { + if (observations.length < 2 || !(coherent || rng.chance(0.7))) { + continue; + } + const targets = observations.filter( + (candidate) => candidate.role !== observation.role, + ); + const dependencyCount = rng.between(1, Math.min(2, targets.length)); + const names = rng.sample(DEPENDENCY_NAME_POOL, dependencyCount); + /** @type {Array>} */ + const dependencies = []; + names.forEach((name, slot) => { + const target = targets[slot % targets.length]; + if (target === undefined) { + return; + } + const exactResource = coherent || rng.chance(0.8); + /** @type {Record} */ + const dependency = { + name, + resource: exactResource + ? target.resource + : randomResource(rng, 90 + slot), + relation: "exact", + provenance: rng.pick(PROVENANCE_POOL), + }; + let versionMatches = false; + if (coherent && target.version !== null) { + dependency["version"] = target.version; + versionMatches = true; + } else { + const versionRoll = rng.below(10); + if (versionRoll < 5 && target.version !== null) { + dependency["version"] = target.version; + versionMatches = true; + } else if (versionRoll < 8) { + const chosen = rng.pick(VERSION_POOL); + dependency["version"] = chosen; + versionMatches = chosen === target.version; + } + } + dependencies.push(dependency); + observation.links.push({ + name, + targetRole: target.role, + coherent: exactResource && versionMatches, + }); + }); + if (dependencies.length > 0) { + /** @type {Record} */ + const witness = /** @type {Record} */ ( + observation.record["witness"] + ); + witness["dependencies"] = dependencies; + } + } + + /** @type {Array<{ observation: typeof observations[number], link: { name: string, targetRole: string, coherent: boolean } }>} */ + const coherentLinks = []; + for (const observation of observations) { + for (const link of observation.links) { + if (link.coherent) { + coherentLinks.push({ observation, link }); + } + } + } + + /** @type {Array>} */ + const requirements = []; + const requirementCount = rng.between(1, 5); + for (let index = 0; index < requirementCount; index += 1) { + const id = `req-${index}`; + const description = rng.chance(0.3) + ? `要件 ${index}` + : `Requirement ${index}`; + let roll = rng.below(3); + if (coherent && roll === 0 && coherentLinks.length === 0) { + roll = 2; + } + /** @type {Record} */ + let requirement; + if (roll === 0 && coherent) { + const chosen = rng.pick(coherentLinks); + requirement = { + id, + type: "dependency", + description, + dependentRole: chosen.observation.role, + targetRole: chosen.link.targetRole, + dependencyName: chosen.link.name, + }; + } else if (roll === 0) { + const dependent = rng.pick(observations); + const target = rng.pick(observations); + const knownName = + dependent.links.length > 0 && rng.chance(0.75) + ? rng.pick(dependent.links).name + : rng.pick(DEPENDENCY_NAME_POOL); + requirement = { + id, + type: "dependency", + description, + dependentRole: rng.chance(0.9) ? dependent.role : "missing-role", + targetRole: rng.chance(0.9) ? target.role : "absent-role", + dependencyName: knownName, + }; + } else if (roll === 1 && observations.length >= 2) { + const chosen = rng.sample( + observations.map((observation) => observation.role), + rng.between(2, Math.min(3, observations.length)), + ); + const fromMs = coherent + ? decisionTimeMs - 3_600_000 + : decisionTimeMs - rng.between(0, 7_200_000); + requirement = { + id, + type: "common_valid_time", + description, + roles: chosen, + within: { + from: timestamp(fromMs), + until: rng.chance(0.15) + ? null + : timestamp( + fromMs + (coherent ? 1_800_000 : rng.between(1, 5_400_000)), + ), + }, + }; + } else { + const observation = rng.pick(observations); + /** @type {Array<{ path: string[], node: unknown }>} */ + const paths = []; + collectPaths(observation.value, [], paths); + if (paths.length > 0 && (coherent || rng.chance(0.7))) { + const chosen = rng.pick(paths); + requirement = { + id, + type: "value_equals", + description, + role: observation.role, + path: chosen.path, + expected: + coherent || rng.chance(0.7) ? chosen.node : randomValue(rng, 2), + }; + } else { + requirement = { + id, + type: "value_equals", + description, + role: rng.chance(0.9) ? observation.role : "unbound-role", + path: [rng.pick(KEY_POOL) || "status"], + expected: randomValue(rng, 2), + }; + } + } + if (index > 0 && rng.chance(0.25)) { + requirement["required"] = false; + } + requirements.push(requirement); + } + + const contract = { + id: rng.chance(0.2) ? "契約-1" : "deploy-current-tested-release", + version: String(rng.between(1, 9)), + decisionTime: timestamp(decisionTimeMs), + assumptions: { + clockModel: "trusted_normalized", + intervalModel: "half_open", + metadataModel: "honest_but_possibly_incomplete", + }, + requirements: rng.chance(0.5) ? rng.shuffled(requirements) : requirements, + }; + + const observationRecords = observations.map( + (observation) => observation.record, + ); + const input = { + protocolVersion: "0.1", + contract, + observations: rng.chance(0.5) + ? rng.shuffled(observationRecords) + : observationRecords, + }; + + return { + text: toJsonText(input, { indent: rng.pick([0, 2, 4]) }), + }; +} + +/** + * @param {string} seed + * @param {number} index + * @returns {{ id: string, category: "random", text: string, expect: "oracle" }} + */ +export function generateRandomCase(seed, index) { + const rng = new DeterministicRandom(`${seed}:${index}`); + const built = buildInput(rng); + return { + id: `random/${String(index).padStart(4, "0")}`, + category: "random", + text: built.text, + expect: "oracle", + }; +} + +/** + * @param {string} seed + * @param {number} count + * @returns {Array<{ id: string, category: "random", text: string, expect: "oracle" }>} + */ +export function generateRandomCases(seed, count) { + const cases = []; + for (let index = 0; index < count; index += 1) { + cases.push(generateRandomCase(seed, index)); + } + return cases; +} diff --git a/scripts/differential/jsconfig.json b/scripts/differential/jsconfig.json new file mode 100644 index 0000000..aab6fb6 --- /dev/null +++ b/scripts/differential/jsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "module": "NodeNext", + "moduleResolution": "NodeNext", + "noEmit": true, + "skipLibCheck": false, + "strict": true, + "target": "ES2023", + "types": ["node"] + }, + "include": ["../differential.mjs", "./*.mjs"] +} diff --git a/scripts/differential/json-text.mjs b/scripts/differential/json-text.mjs new file mode 100644 index 0000000..313cff9 --- /dev/null +++ b/scripts/differential/json-text.mjs @@ -0,0 +1,194 @@ +/** + * A JSON writer that preserves the exact lexical form of chosen tokens. + * + * `JSON.stringify` normalizes numbers before any port can observe them, so + * `1e-400` becomes `0` and `-0` becomes `0`. The differential corpus has to send + * the original lexeme to every CLI, so cases are built from these node types and + * serialized here instead. + */ + +/** A literal JSON token that is emitted exactly as written. */ +export class RawJson { + /** + * @param {string} text Valid JSON token text, for example `1e-400`. + */ + constructor(text) { + this.text = text; + } +} + +/** + * An object whose members are emitted in the given order, allowing repeated + * member names for transport-level cases. + */ +export class OrderedMembers { + /** + * @param {ReadonlyArray} entries + */ + constructor(entries) { + /** @type {Array<[string, unknown]>} */ + this.entries = entries.map(([key, value]) => [key, value]); + } +} + +/** + * @param {string} text + * @returns {RawJson} + */ +export function raw(text) { + return new RawJson(text); +} + +/** + * @param {ReadonlyArray} entries + * @returns {OrderedMembers} + */ +export function members(entries) { + return new OrderedMembers(entries); +} + +const ESCAPES = new Map([ + ['"', '\\"'], + ["\\", "\\\\"], + ["\b", "\\b"], + ["\f", "\\f"], + ["\n", "\\n"], + ["\r", "\\r"], + ["\t", "\\t"], +]); + +/** + * Escapes a JavaScript string into a JSON string token. + * + * Non-ASCII characters are emitted literally so the corpus exercises real UTF-8 + * transport bytes rather than `\u` escapes. + * + * @param {string} value + * @returns {string} + */ +export function encodeJsonString(value) { + let out = '"'; + for (const character of value) { + const escape = ESCAPES.get(character); + if (escape !== undefined) { + out += escape; + continue; + } + const code = character.codePointAt(0) ?? 0; + if (code < 0x20 || code === 0x7f) { + out += `\\u${code.toString(16).padStart(4, "0")}`; + continue; + } + out += character; + } + return `${out}"`; +} + +/** + * Serializes a node tree to JSON text. + * + * @param {unknown} value + * @param {{ indent?: number }} [options] + * @returns {string} + */ +export function toJsonText(value, options = {}) { + const indent = options.indent ?? 2; + return write(value, indent, 0); +} + +/** + * @param {unknown} value + * @param {number} indent + * @param {number} depth + * @returns {string} + */ +function write(value, indent, depth) { + if (value instanceof RawJson) { + return value.text; + } + if (value === null) { + return "null"; + } + if (typeof value === "boolean") { + return value ? "true" : "false"; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new TypeError(`cannot serialize non-finite number ${value}`); + } + if (Object.is(value, -0)) { + throw new TypeError("use raw(\"-0\") so the lexeme survives serialization"); + } + return String(value); + } + if (typeof value === "string") { + return encodeJsonString(value); + } + if (Array.isArray(value)) { + return writeItems( + value.map((item) => write(item, indent, depth + 1)), + "[", + "]", + indent, + depth, + ); + } + if (value instanceof OrderedMembers) { + return writeItems( + value.entries.map( + ([key, item]) => + `${encodeJsonString(key)}:${indent > 0 ? " " : ""}${write(item, indent, depth + 1)}`, + ), + "{", + "}", + indent, + depth, + ); + } + if (typeof value === "object") { + return writeItems( + Object.entries(value).map( + ([key, item]) => + `${encodeJsonString(key)}:${indent > 0 ? " " : ""}${write(item, indent, depth + 1)}`, + ), + "{", + "}", + indent, + depth, + ); + } + throw new TypeError(`cannot serialize ${typeof value}`); +} + +/** + * @param {string[]} pieces + * @param {string} open + * @param {string} close + * @param {number} indent + * @param {number} depth + * @returns {string} + */ +function writeItems(pieces, open, close, indent, depth) { + if (pieces.length === 0) { + return `${open}${close}`; + } + if (indent <= 0) { + return `${open}${pieces.join(",")}${close}`; + } + const inner = " ".repeat(indent * (depth + 1)); + const outer = " ".repeat(indent * depth); + return `${open}\n${inner}${pieces.join(`,\n${inner}`)}\n${outer}${close}`; +} + +/** + * Converts a node tree into plain JSON data by resolving raw tokens. + * + * Used by self-checks to confirm that a generated case is the JSON value the + * generator intended. + * + * @param {unknown} value + * @returns {unknown} + */ +export function toJsonData(value) { + return JSON.parse(toJsonText(value, { indent: 0 })); +} diff --git a/scripts/differential/ports.mjs b/scripts/differential/ports.mjs new file mode 100644 index 0000000..1e576f7 --- /dev/null +++ b/scripts/differential/ports.mjs @@ -0,0 +1,352 @@ +/** + * Port discovery, one-time builds, and per-case CLI execution. + * + * Each non-TypeScript port is built exactly once into the temporary workspace + * (or its conventional project output directory) and then spawned per case. + * Executables are overridable through environment variables so a developer + * machine can point at a private toolchain. + */ + +import { execFile } from "node:child_process"; +import { access, constants } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +/** Maximum bytes captured from one CLI invocation. */ +const MAX_BUFFER = 64 * 1024 * 1024; + +/** Maximum time allowed for a one-time port build or readiness probe. */ +const BUILD_TIMEOUT_MS = 5 * 60 * 1000; + +const IS_WINDOWS = process.platform === "win32"; + +/** + * @param {string} path + * @returns {Promise} + */ +async function exists(path) { + try { + await access(path, constants.F_OK); + return true; + } catch { + return false; + } +} + +/** + * @param {string} command + * @param {string[]} args + * @param {{ cwd?: string, env?: NodeJS.ProcessEnv, timeoutMs?: number }} [options] + * @returns {Promise} + */ +async function run(command, args, options = {}) { + const timeoutMs = options.timeoutMs ?? BUILD_TIMEOUT_MS; + try { + const { stdout } = await execFileAsync(command, args, { + cwd: options.cwd, + env: options.env ?? process.env, + maxBuffer: MAX_BUFFER, + timeout: timeoutMs, + killSignal: "SIGKILL", + windowsHide: true, + }); + return stdout.toString(); + } catch (error) { + if ( + error !== null && + typeof error === "object" && + "killed" in error && + error.killed === true + ) { + throw new Error( + `${command} timed out after ${timeoutMs}ms while preparing a port`, + { cause: error }, + ); + } + throw error; + } +} + +/** + * Describes how one implementation is invoked. + * + * @typedef {object} PortRunner + * @property {string} id + * @property {string} label + * @property {string} command + * @property {string[]} baseArgs + * @property {string} [cwd] + * @property {NodeJS.ProcessEnv} [env] + * @property {number} timeoutMs + * @property {string} description + */ + +/** + * @typedef {object} PortOutcome + * @property {number} status Process exit code. + * @property {string} stdout + * @property {string} stderr + * @property {"result" | "error" | "unusable"} kind + * @property {unknown} [result] Parsed verification result when `kind` is `result`. + * @property {string} [code] Stable error code when `kind` is `error`. + * @property {string} [message] Human error message when `kind` is `error`. + * @property {string} [failure] Why the output could not be interpreted. + */ + +/** + * Builds every port once and returns their runners. + * + * @param {{ repoRoot: string, workspace: string, timeoutMs: number, log: (line: string) => void }} context + * @returns {Promise} + */ +export async function prepareRunners(context) { + const { repoRoot, workspace, timeoutMs, log } = context; + + /** @type {PortRunner[]} */ + const runners = []; + + const nodeExecutable = process.env["WORLDCUT_NODE"] ?? process.execPath; + const cliPath = join(repoRoot, "dist", "cli.js"); + if (!(await exists(cliPath))) { + throw new Error( + `${cliPath} is missing. Run "npm run build" before the differential suite.`, + ); + } + runners.push({ + id: "typescript", + label: "TypeScript", + command: nodeExecutable, + baseArgs: [cliPath, "--full"], + timeoutMs, + description: `${nodeExecutable} dist/cli.js --full`, + }); + + const goExecutable = process.env["WORLDCUT_GO"] ?? "go"; + const goPortRoot = join(repoRoot, "ports", "go"); + const goBinary = join( + workspace, + IS_WINDOWS ? "worldcut-go.exe" : "worldcut-go", + ); + log(`building Go CLI with ${goExecutable}`); + await run(goExecutable, ["build", "-o", goBinary, "./cmd/worldcut-go"], { + cwd: goPortRoot, + }); + runners.push({ + id: "go", + label: "Go", + command: goBinary, + baseArgs: [], + timeoutMs, + description: "ports/go/cmd/worldcut-go", + }); + + const pythonExecutable = process.env["WORLDCUT_PYTHON"] ?? "python"; + log(`checking Python port with ${pythonExecutable}`); + await run(pythonExecutable, [ + "-c", + "import worldcut; assert worldcut.ENGINE_VERSION", + ]); + runners.push({ + id: "python", + label: "Python", + command: pythonExecutable, + baseArgs: ["-m", "worldcut.cli"], + timeoutMs, + description: `${pythonExecutable} -m worldcut.cli`, + }); + + const dotnetExecutable = process.env["WORLDCUT_DOTNET"] ?? "dotnet"; + const framework = process.env["WORLDCUT_DOTNET_FRAMEWORK"] ?? "net8.0"; + const dotnetPortRoot = join(repoRoot, "ports", "dotnet"); + const project = join("src", "WorldCut.Tool", "WorldCut.Tool.csproj"); + log(`building .NET CLI (${framework}) with ${dotnetExecutable}`); + await run( + dotnetExecutable, + [ + "build", + project, + "--configuration", + "Release", + "--framework", + framework, + "-p:RestoreLockedMode=true", + ], + { + cwd: dotnetPortRoot, + env: { + ...process.env, + DOTNET_NOLOGO: "1", + DOTNET_CLI_TELEMETRY_OPTOUT: "1", + }, + }, + ); + const dotnetOutput = join( + dotnetPortRoot, + "src", + "WorldCut.Tool", + "bin", + "Release", + framework, + ); + const assembly = join(dotnetOutput, "WorldCut.Tool.dll"); + if (!(await exists(assembly))) { + throw new Error(`the .NET build produced no CLI under ${dotnetOutput}`); + } + // The built apphost resolves its runtime through DOTNET_ROOT or a + // machine-wide install, which may not be the host that produced the build. + // Running the assembly through the selected muxer keeps the harness on one + // runtime no matter where the SDK lives. + runners.push({ + id: "dotnet", + label: ".NET", + command: dotnetExecutable, + baseArgs: ["exec", assembly], + timeoutMs, + description: `${dotnetExecutable} exec WorldCut.Tool.dll (${framework})`, + }); + + return runners; +} + +/** + * Runs one port against one input file and classifies its output. + * + * @param {PortRunner} runner + * @param {string} inputPath + * @returns {Promise} + */ +export function runPort(runner, inputPath) { + return new Promise((resolve) => { + execFile( + runner.command, + [...runner.baseArgs, inputPath], + { + cwd: runner.cwd, + env: runner.env ?? process.env, + maxBuffer: MAX_BUFFER, + timeout: runner.timeoutMs, + killSignal: "SIGKILL", + windowsHide: true, + encoding: "buffer", + }, + (error, stdoutBuffer, stderrBuffer) => { + const stdout = stdoutBuffer.toString("utf8"); + const stderr = stderrBuffer.toString("utf8"); + /** @type {number} */ + let status; + if (error === null) { + status = 0; + } else if (typeof error.code === "number") { + status = error.code; + } else { + const failure = + error.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER" + ? `${runner.id} exceeded the ${MAX_BUFFER}-byte output limit` + : error.killed + ? `${runner.id} timed out after ${runner.timeoutMs}ms` + : `${runner.id} could not be executed: ${error.message}`; + resolve({ + status: -1, + stdout, + stderr, + kind: "unusable", + failure, + }); + return; + } + resolve(classify(runner, status, stdout, stderr)); + }, + ); + }); +} + +/** + * @param {PortRunner} runner + * @param {number} status + * @param {string} stdout + * @param {string} stderr + * @returns {PortOutcome} + */ +function classify(runner, status, stdout, stderr) { + if (status === 0) { + try { + return { + status, + stdout, + stderr, + kind: "result", + result: JSON.parse(stdout), + }; + } catch (error) { + return { + status, + stdout, + stderr, + kind: "unusable", + failure: `${runner.id} exited 0 but printed unparsable JSON: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + } + + const envelope = parseEnvelope(stderr); + if (envelope === null) { + return { + status, + stdout, + stderr, + kind: "unusable", + failure: `${runner.id} exited ${status} without a stable error envelope`, + }; + } + return { + status, + stdout, + stderr, + kind: "error", + code: envelope.code, + message: envelope.message, + }; +} + +/** + * @param {string} stderr + * @returns {{ code: string, message: string } | null} + */ +function parseEnvelope(stderr) { + const trimmed = stderr.trim(); + if (trimmed.length === 0) { + return null; + } + const lines = trimmed.split(/\r?\n/); + const last = lines[lines.length - 1]; + if (last === undefined) { + return null; + } + try { + const parsed = JSON.parse(last); + if ( + parsed !== null && + typeof parsed === "object" && + "error" in parsed && + parsed.error !== null && + typeof parsed.error === "object" && + typeof (/** @type {{ code?: unknown }} */ (parsed.error).code) === "string" + ) { + const envelope = /** @type {{ code: string, message?: unknown }} */ ( + parsed.error + ); + return { + code: envelope.code, + message: + typeof envelope.message === "string" ? envelope.message : "", + }; + } + return null; + } catch { + return null; + } +} diff --git a/scripts/differential/prng.mjs b/scripts/differential/prng.mjs new file mode 100644 index 0000000..7cd059b --- /dev/null +++ b/scripts/differential/prng.mjs @@ -0,0 +1,160 @@ +/** + * Deterministic pseudo-random source for the cross-language differential + * harness. + * + * The generator must produce byte-identical corpora from the same seed on every + * platform, so it uses only 32-bit integer arithmetic and never touches + * `Math.random`, the clock, or the environment. + */ + +/** + * Expands an arbitrary seed string into four 32-bit state words (cyrb128). + * + * @param {string} seed + * @returns {[number, number, number, number]} + */ +export function expandSeed(seed) { + let h1 = 1779033703; + let h2 = 3144134277; + let h3 = 1013904242; + let h4 = 2773480762; + for (let index = 0; index < seed.length; index += 1) { + const code = seed.charCodeAt(index); + h1 = h2 ^ Math.imul(h1 ^ code, 597399067); + h2 = h3 ^ Math.imul(h2 ^ code, 2869860233); + h3 = h4 ^ Math.imul(h3 ^ code, 951274213); + h4 = h1 ^ Math.imul(h4 ^ code, 2716044179); + } + h1 = Math.imul(h3 ^ (h1 >>> 18), 597399067); + h2 = Math.imul(h4 ^ (h2 >>> 22), 2869860233); + h3 = Math.imul(h1 ^ (h3 >>> 17), 951274213); + h4 = Math.imul(h2 ^ (h4 >>> 19), 2716044179); + return [ + (h1 ^ h2 ^ h3 ^ h4) >>> 0, + (h2 ^ h1) >>> 0, + (h3 ^ h1) >>> 0, + (h4 ^ h1) >>> 0, + ]; +} + +/** + * A small deterministic random helper built on the sfc32 generator. + */ +export class DeterministicRandom { + /** + * @param {string} seed + */ + constructor(seed) { + const [a, b, c, d] = expandSeed(String(seed)); + this.a = a; + this.b = b; + this.c = c; + this.d = d; + for (let index = 0; index < 12; index += 1) { + this.next(); + } + } + + /** + * @returns {number} The next 32-bit unsigned integer. + */ + next() { + const t = (((this.a + this.b) | 0) + this.d) | 0; + this.d = (this.d + 1) | 0; + this.a = this.b ^ (this.b >>> 9); + this.b = (this.c + (this.c << 3)) | 0; + this.c = (this.c << 21) | (this.c >>> 11); + this.c = (this.c + t) | 0; + return t >>> 0; + } + + /** + * @returns {number} A float in `[0, 1)`. + */ + float() { + return this.next() / 4294967296; + } + + /** + * @param {number} bound Exclusive upper bound, must be a positive integer. + * @returns {number} An integer in `[0, bound)`. + */ + below(bound) { + if (!Number.isSafeInteger(bound) || bound <= 0) { + throw new RangeError( + `bound must be a positive integer, received ${bound}`, + ); + } + return this.next() % bound; + } + + /** + * @param {number} min Inclusive lower bound. + * @param {number} max Inclusive upper bound. + * @returns {number} + */ + between(min, max) { + if (max < min) { + throw new RangeError(`max ${max} is below min ${min}`); + } + return min + this.below(max - min + 1); + } + + /** + * @param {number} probability A value in `[0, 1]`. + * @returns {boolean} + */ + chance(probability) { + return this.float() < probability; + } + + /** + * @template T + * @param {readonly T[]} items + * @returns {T} + */ + pick(items) { + if (items.length === 0) { + throw new RangeError("cannot pick from an empty list"); + } + const chosen = items[this.below(items.length)]; + if (chosen === undefined) { + throw new Error("deterministic pick returned nothing"); + } + return chosen; + } + + /** + * Returns a shuffled copy using a Fisher-Yates pass. + * + * @template T + * @param {readonly T[]} items + * @returns {T[]} + */ + shuffled(items) { + const copy = [...items]; + for (let index = copy.length - 1; index > 0; index -= 1) { + const swap = this.below(index + 1); + const left = copy[index]; + const right = copy[swap]; + if (left === undefined || right === undefined) { + throw new Error("deterministic shuffle lost an element"); + } + copy[index] = right; + copy[swap] = left; + } + return copy; + } + + /** + * Picks `count` distinct entries, preserving deterministic order. + * + * @template T + * @param {readonly T[]} items + * @param {number} count + * @returns {T[]} + */ + sample(items, count) { + return this.shuffled(items).slice(0, Math.min(count, items.length)); + } +} diff --git a/scripts/differential/self-check.mjs b/scripts/differential/self-check.mjs new file mode 100644 index 0000000..36617e6 --- /dev/null +++ b/scripts/differential/self-check.mjs @@ -0,0 +1,435 @@ +/** + * Deterministic self-checks for the differential harness. + * + * These run before any port is executed. If the generator, the raw-lexeme + * writer, or the comparison logic ever regresses, the suite fails immediately + * instead of silently comparing a weaker corpus. + */ + +import { DeterministicRandom, expandSeed } from "./prng.mjs"; +import { + encodeJsonString, + members, + raw, + toJsonData, + toJsonText, +} from "./json-text.mjs"; +import { diffJson, jsonEquals, toRawOutcome, DIGEST_PATTERN } from "./compare.mjs"; +import { selectCases } from "./corpus.mjs"; +import { generateRandomCase } from "./generate.mjs"; + +/** + * @param {boolean} condition + * @param {string} message + */ +function check(condition, message) { + if (!condition) { + throw new Error(`harness self-check failed: ${message}`); + } +} + +/** + * @returns {number} The number of assertions performed. + */ +function checkPrng() { + let assertions = 0; + + const first = new DeterministicRandom("seed-a"); + const second = new DeterministicRandom("seed-a"); + const third = new DeterministicRandom("seed-b"); + const a = Array.from({ length: 64 }, () => first.next()); + const b = Array.from({ length: 64 }, () => second.next()); + const c = Array.from({ length: 64 }, () => third.next()); + check(a.join() === b.join(), "the same seed must replay the same sequence"); + check(a.join() !== c.join(), "different seeds must diverge"); + assertions += 2; + + check( + expandSeed("seed-a").join() === expandSeed("seed-a").join(), + "seed expansion must be pure", + ); + assertions += 1; + + const rng = new DeterministicRandom("bounds"); + for (let index = 0; index < 500; index += 1) { + const value = rng.below(7); + check(value >= 0 && value < 7, "below() must stay inside its bound"); + const ranged = rng.between(-3, 3); + check(ranged >= -3 && ranged <= 3, "between() must stay inside its bounds"); + } + assertions += 2; + + const source = [1, 2, 3, 4, 5, 6, 7, 8]; + const shuffled = new DeterministicRandom("shuffle").shuffled(source); + check(shuffled.length === source.length, "shuffle must preserve length"); + check( + [...shuffled].sort((left, right) => left - right).join() === source.join(), + "shuffle must preserve elements", + ); + check(source.join() === "1,2,3,4,5,6,7,8", "shuffle must not mutate its input"); + check( + new DeterministicRandom("shuffle").shuffled(source).join() === + shuffled.join(), + "shuffle must be deterministic", + ); + assertions += 4; + + const sample = new DeterministicRandom("sample").sample(source, 3); + check(sample.length === 3, "sample must honour its count"); + check(new Set(sample).size === 3, "sample must be distinct"); + assertions += 2; + + return assertions; +} + +/** + * @returns {number} + */ +function checkJsonText() { + let assertions = 0; + + check( + toJsonText(raw("1e-400"), { indent: 0 }) === "1e-400", + "raw number lexemes must survive serialization", + ); + check( + toJsonText(raw("-0"), { indent: 0 }) === "-0", + "negative zero must survive serialization", + ); + check( + JSON.stringify(-0) === "0", + "JSON.stringify still erases negative zero, which is why raw() exists", + ); + assertions += 3; + + const duplicated = toJsonText( + members([ + ["a", 1], + ["a", 2], + ]), + { indent: 0 }, + ); + check( + duplicated === '{"a":1,"a":2}', + `duplicate member names must be preserved, got ${duplicated}`, + ); + assertions += 1; + + const ordered = toJsonText( + members([ + ["z", 1], + ["a", 2], + ]), + { indent: 0 }, + ); + check(ordered === '{"z":1,"a":2}', "declared member order must be preserved"); + assertions += 1; + + check( + encodeJsonString('a"b\\c\nd\te\u0000f') === '"a\\"b\\\\c\\nd\\te\\u0000f"', + "string escaping must be JSON-legal", + ); + check( + encodeJsonString("Ω 𝄞") === '"Ω 𝄞"', + "non-ASCII characters stay literal so transport bytes are real UTF-8", + ); + assertions += 2; + + const nested = members([ + ["outer", [1, members([["inner", "x"]]), null, true]], + ]); + check( + jsonEquals(toJsonData(nested), { outer: [1, { inner: "x" }, null, true] }), + "node trees must resolve to the intended JSON data", + ); + check( + toJsonText(nested, { indent: 2 }).includes("\n"), + "indentation must be applied when requested", + ); + check( + !toJsonText(nested, { indent: 0 }).includes("\n"), + "compact output must have no line breaks", + ); + assertions += 3; + + let rejected = false; + try { + toJsonText(-0); + } catch { + rejected = true; + } + check(rejected, "a bare -0 must be rejected so lexemes are never lost"); + assertions += 1; + + return assertions; +} + +/** + * @returns {number} + */ +function checkCompare() { + let assertions = 0; + + check( + jsonEquals({ a: 1, b: [1, 2] }, { b: [1, 2], a: 1 }), + "member order must be ignored", + ); + check(jsonEquals(0, -0), "negative zero equals zero under worldcut-json-v1"); + check(!jsonEquals([1, 2], [2, 1]), "array order must be significant"); + check(!jsonEquals({ a: 1 }, { a: 1, b: 2 }), "extra members must be reported"); + check(!jsonEquals(1, "1"), "type differences must be reported"); + check(!jsonEquals(null, false), "null and false must differ"); + assertions += 6; + + const differences = diffJson( + { plan: { actions: [{ cost: 3 }] } }, + { plan: { actions: [{ cost: 4 }] } }, + ); + check(differences.length === 1, "one value difference must be reported once"); + check( + differences[0]?.path === "/plan/actions/0/cost", + `difference path must be a JSON pointer, got ${differences[0]?.path}`, + ); + assertions += 2; + + const escaped = diffJson({ "a/b~c": 1 }, { "a/b~c": 2 }); + check( + escaped[0]?.path === "/a~1b~0c", + `pointer segments must be escaped, got ${escaped[0]?.path}`, + ); + assertions += 1; + + const many = diffJson( + { a: 1, b: 1, c: 1, d: 1 }, + { a: 2, b: 2, c: 2, d: 2 }, + { limit: 2 }, + ); + check(many.length === 2, "the difference limit must be honoured"); + assertions += 1; + + check( + toRawOutcome("WORLDCUT_INVALID_INPUT") === "WORLDCUT_INVALID_INPUT", + "validation failures keep their code", + ); + check( + toRawOutcome("WORLDCUT_INVALID_JSON") === "PARSE_ERROR", + "parser failures map onto the PARSE_ERROR outcome", + ); + for (const code of [ + "WORLDCUT_RUNTIME_ERROR", + "WORLDCUT_FILE_READ_FAILED", + "WORLDCUT_INVALID_ARGUMENT", + ]) { + check( + toRawOutcome(code) === null, + `${code} must not count as a parser rejection`, + ); + } + assertions += 3; + + check( + DIGEST_PATTERN.test("f".repeat(64)) && + !DIGEST_PATTERN.test("F".repeat(64)) && + !DIGEST_PATTERN.test("f".repeat(63)), + "the digest pattern must require 64 lowercase hex characters", + ); + assertions += 1; + + return assertions; +} + +/** + * @returns {number} + */ +function checkGenerator() { + let assertions = 0; + + for (const index of [0, 1, 7, 42, 199]) { + const first = generateRandomCase("self-check", index); + const second = generateRandomCase("self-check", index); + check( + first.text === second.text, + `random case ${index} must be reproducible from its seed`, + ); + check( + first.id === second.id, + `random case ${index} must have a stable identifier`, + ); + const parsed = JSON.parse(first.text); + check( + parsed.protocolVersion === "0.1", + `random case ${index} must declare protocol 0.1`, + ); + check( + Array.isArray(parsed.observations) && parsed.observations.length >= 2, + `random case ${index} must bind at least two roles`, + ); + check( + Array.isArray(parsed.contract.requirements) && + parsed.contract.requirements.length >= 1, + `random case ${index} must declare a requirement`, + ); + check( + parsed.contract.requirements.some( + (/** @type {{ required?: boolean }} */ requirement) => + requirement.required !== false, + ), + `random case ${index} must keep at least one required requirement`, + ); + const roles = parsed.observations.map( + (/** @type {{ role: string }} */ observation) => observation.role, + ); + check( + new Set(roles).size === roles.length, + `random case ${index} must not repeat a role`, + ); + const ids = parsed.observations.map( + (/** @type {{ id: string }} */ observation) => observation.id, + ); + check( + new Set(ids).size === ids.length, + `random case ${index} must not repeat an observation id`, + ); + } + assertions += 7; + + const seedA = generateRandomCase("seed-a", 3).text; + const seedB = generateRandomCase("seed-b", 3).text; + check(seedA !== seedB, "different seeds must produce different cases"); + assertions += 1; + + const sampled = Array.from({ length: 120 }, (_, index) => + generateRandomCase("coverage", index).text, + ); + check( + new Set(sampled).size > sampled.length / 2, + "the generator must not collapse onto a handful of documents", + ); + const combined = sampled.join("\n"); + for (const feature of [ + "1e-400", + "-0", + "value_equals", + "common_valid_time", + "dependency", + "dependencies", + "validity", + '"required": false', + '"required":false', + ]) { + check( + combined.includes(feature), + `the generated corpus must exercise ${feature}`, + ); + } + for (const codePoint of ["Ω", "日", "𝄞", "Ｚ"]) { + check( + combined.includes(codePoint), + `the generated corpus must exercise the ${codePoint} code point`, + ); + } + assertions += 3; + + return assertions; +} + +/** + * @param {import("./corpus.mjs").DifferentialCase[]} cases + * @param {number} randomCount + * @returns {number} + */ +function checkCorpus(cases, randomCount) { + let assertions = 0; + + const ids = cases.map((entry) => entry.id); + check(new Set(ids).size === ids.length, "case identifiers must be unique"); + assertions += 1; + + const categories = new Set(cases.map((entry) => entry.category)); + const required = [ + "golden", + "invalid", + "raw", + "example", + "edge", + "transport", + ]; + if (randomCount > 0) { + required.push("random"); + } + for (const name of required) { + check( + categories.has(name), + `the corpus must include the ${name} category`, + ); + } + assertions += 1; + + for (const entry of cases) { + check( + Buffer.isBuffer(entry.bytes), + `${entry.id} must carry transport bytes`, + ); + if (entry.expect === "code") { + check( + typeof entry.code === "string" && entry.code.length > 0, + `${entry.id} must name the expected error code`, + ); + } + if (entry.expect === "transport") { + check( + Array.isArray(entry.allowed) && entry.allowed.length > 0, + `${entry.id} must list accepted outcomes`, + ); + } + if (entry.sameDigestAs !== undefined) { + check( + ids.includes(entry.sameDigestAs), + `${entry.id} references unknown twin ${entry.sameDigestAs}`, + ); + } + } + assertions += 1; + + const underflow = cases.filter((entry) => + entry.bytes.includes("1e-400"), + ); + check( + underflow.length > 0, + "the corpus must send a finite underflow lexeme to every port", + ); + assertions += 1; + + const digestPair = cases.find((entry) => entry.sameDigestAs !== undefined); + check(digestPair !== undefined, "the corpus must contain a digest-equivalence pair"); + if (digestPair !== undefined) { + const selected = selectCases(cases, { + category: null, + only: digestPair.id, + }); + check( + selected.some((entry) => entry.id === digestPair.sameDigestAs), + "selecting one digest-equivalence case must include its twin", + ); + } + assertions += 1; + + return assertions; +} + +/** + * Runs every self-check. + * + * @param {import("./corpus.mjs").DifferentialCase[]} cases + * @param {number} randomCount + * @returns {{ assertions: number }} + */ +export function runSelfChecks(cases, randomCount) { + const assertions = + checkPrng() + + checkJsonText() + + checkCompare() + + checkGenerator() + + checkCorpus(cases, randomCount); + return { assertions }; +} diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index e3d8869..b9e1cc1 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -54,6 +54,7 @@ try { "docs/INTEGRATIONS.md", "docs/AGENTIC_DATA_KERNEL.md", "docs/VALIDATION.md", + "docs/DIFFERENTIAL.md", "spec/0.1/PROTOCOL.md", "spec/0.1/CANONICALIZATION.md", "spec/0.1/CONFORMANCE.md", @@ -82,6 +83,7 @@ try { ".github/", "dist/benchmark/", "dist/test/", + "scripts/", "src/", ]) { if ([...packagedPaths].some((path) => path.startsWith(forbiddenPrefix))) { diff --git a/spec/0.1/CONFORMANCE.md b/spec/0.1/CONFORMANCE.md index b5a91f6..0ba12c8 100644 --- a/spec/0.1/CONFORMANCE.md +++ b/spec/0.1/CONFORMANCE.md @@ -19,6 +19,16 @@ Exact result construction and digest material are defined in Human summaries, acquisition actions, ordering, and digests are included in the verification vectors. Implementations cannot substitute equivalent wording. +## Cross-implementation checking + +Passing the vectors is the conformance requirement. It is not evidence that two +implementations still agree on inputs no vector covers. + +This repository additionally runs every implementation over one shared corpus of +golden, edge, invalid, malformed, and seeded random inputs and compares the +complete results. That suite is a development and CI gate, not part of this +normative specification; see `docs/DIFFERENTIAL.md`. + ## Updating vectors Protocol behavior must change before vectors change. Update the normative diff --git a/src/cli.ts b/src/cli.ts index 9a5508a..9efddd1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -90,9 +90,9 @@ async function main(): Promise { } const inputPath = resolve(options.inputPath); - let source: string; + let bytes: Buffer; try { - source = await readFile(inputPath, "utf8"); + bytes = await readFile(inputPath); } catch (error) { throw new WorldCutError( "WORLDCUT_FILE_READ_FAILED", @@ -100,6 +100,24 @@ async function main(): Promise { { cause: error }, ); } + // Node's lossy UTF-8 decoding would substitute U+FFFD for malformed bytes and + // then verify the corrupted evidence. Every other WorldCut port rejects + // transport bytes that are not valid UTF-8, so this one does too. + // `ignoreBOM` keeps a leading U+FEFF in the string, where JSON.parse rejects + // it, instead of silently accepting a byte-order mark the other ports refuse. + let source: string; + try { + source = new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true, + }).decode(bytes); + } catch (error) { + throw new WorldCutError( + "WORLDCUT_INVALID_JSON", + `${inputPath} is not valid UTF-8`, + { cause: error }, + ); + } let input: VerificationInput; try { input = JSON.parse(source) as VerificationInput; diff --git a/src/test/differential-harness.test.ts b/src/test/differential-harness.test.ts new file mode 100644 index 0000000..c46d936 --- /dev/null +++ b/src/test/differential-harness.test.ts @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; +import test from "node:test"; + +/** + * The differential suite itself needs Go, Python, and .NET toolchains, but its + * deterministic self-checks do not. Running them here keeps a regression in the + * seeded generator, the raw-lexeme writer, or the structural comparison from + * silently weakening the cross-language gate. + */ +test("differential harness self-checks pass", () => { + const result = spawnSync( + process.execPath, + [ + join(process.cwd(), "scripts", "differential.mjs"), + "--self-check-only", + "--count", + "200", + ], + { encoding: "utf8" }, + ); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /self-checks passed/); +}); + +test("differential corpus lists every case category", () => { + const result = spawnSync( + process.execPath, + [ + join(process.cwd(), "scripts", "differential.mjs"), + "--list", + "--count", + "3", + ], + { encoding: "utf8" }, + ); + + assert.equal(result.status, 0, result.stderr); + for (const category of [ + "golden", + "invalid", + "raw", + "example", + "edge", + "transport", + "random", + ]) { + assert.match(result.stdout, new RegExp(`${category}=\\d+`), category); + } + assert.match(result.stdout, /edge\/number-underflow-positive/); +}); + +test("differential harness rejects unknown options", () => { + const result = spawnSync( + process.execPath, + [ + join(process.cwd(), "scripts", "differential.mjs"), + "--self-check-only", + "--seeed", + "typo", + ], + { encoding: "utf8" }, + ); + + assert.equal(result.status, 1); + assert.match(result.stderr, /unknown option: --seeed/); +}); + +test("differential harness rejects partially numeric options", () => { + const result = spawnSync( + process.execPath, + [ + join(process.cwd(), "scripts", "differential.mjs"), + "--self-check-only", + "--count", + "500cases", + ], + { encoding: "utf8" }, + ); + + assert.equal(result.status, 1); + assert.match(result.stderr, /--count must be an integer/); +}); + +test("focused differential runs retain digest-equivalence twins", () => { + const result = spawnSync( + process.execPath, + [ + join(process.cwd(), "scripts", "differential.mjs"), + "--list", + "--count", + "0", + "--only", + "edge/unicode-escaped-input", + ], + { encoding: "utf8" }, + ); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /edge\/unicode-escaped-input/); + assert.match(result.stdout, /edge\/unicode-escaped-literal/); + assert.match(result.stdout, /2 cases/); +}); diff --git a/src/test/examples.test.ts b/src/test/examples.test.ts index f6aede3..d499471 100644 --- a/src/test/examples.test.ts +++ b/src/test/examples.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; import type { @@ -95,3 +96,38 @@ test("CLI emits a stable JSON error envelope", () => { assert.equal(error.error.code, "WORLDCUT_FILE_READ_FAILED"); assert.match(error.error.message, /Unable to read/); }); + +test("CLI rejects transport bytes that are not valid UTF-8", () => { + const directory = mkdtempSync(join(tmpdir(), "worldcut-cli-")); + try { + const source = readFileSync( + join(process.cwd(), "examples", "coherent-deployment.json"), + ); + const marker = source.indexOf(Buffer.from("commit-B", "utf8")); + assert.ok(marker > 0, "the fixture no longer contains the splice point"); + const path = join(directory, "invalid-utf8.json"); + writeFileSync( + path, + Buffer.concat([ + source.subarray(0, marker), + Buffer.from([0xff, 0xfe]), + source.subarray(marker), + ]), + ); + + const result = spawnSync( + process.execPath, + [join(process.cwd(), "dist", "cli.js"), path, "--full"], + { encoding: "utf8" }, + ); + + assert.equal(result.status, 1, result.stdout); + const error = JSON.parse(result.stderr) as { + error: { code: string; message: string }; + }; + assert.equal(error.error.code, "WORLDCUT_INVALID_JSON"); + assert.match(error.error.message, /not valid UTF-8/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +});