test: validate fixtures against the fixture schema in CI

.github/workflows/ci.yml

@@ -23,3 +23,4 @@ jobs:
           cache: pnpm
       - run: pnpm install
       - run: pnpm run lint
+      - run: pnpm run test

CONTRIBUTING.md

@@ -50,7 +50,7 @@ For major changes requiring RFC:
 
 ## Contributing Test Fixtures
 
-Test fixtures validate TOON implementations across languages. Add your test to `tests/fixtures/encode/` or `tests/fixtures/decode/`, validate against `tests/fixtures.schema.json`, and submit a PR.
+Test fixtures validate TOON implementations across languages. Add your test to `tests/fixtures/encode/` or `tests/fixtures/decode/`, run `pnpm test` to validate it against `tests/fixtures.schema.json`, and submit a PR.
 
 See [tests/README.md](./tests/README.md) for fixture structure and guidelines.
 

package.json

@@ -32,10 +32,12 @@
   ],
   "scripts": {
     "lint": "eslint .",
+    "test": "node scripts/validate-fixtures.ts",
     "release": "bumpp"
   },
   "devDependencies": {
     "@antfu/eslint-config": "^9.5.1",
+    "ajv": "^8.20.0",
     "bumpp": "^12.3.0",
     "eslint": "^10.11.0"
   }

scripts/validate-fixtures.ts

@@ -0,0 +1,29 @@
+import { readdir, readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import process from 'node:process'
+import Ajv from 'ajv'
+
+const testsDir = join(import.meta.dirname, '../tests')
+const schema = JSON.parse(await readFile(join(testsDir, 'fixtures.schema.json'), 'utf8'))
+const validate = new Ajv({ allErrors: true }).compile(schema)
+
+const fixtureErrors: string[] = []
+for (const category of ['encode', 'decode']) {
+  const categoryDir = join(testsDir, 'fixtures', category)
+  const fileNames = (await readdir(categoryDir)).filter(fileName => fileName.endsWith('.json'))
+  for (const fileName of fileNames) {
+    const fixture = JSON.parse(await readFile(join(categoryDir, fileName), 'utf8'))
+    const fixturePath = `${category}/${fileName}`
+    if (!validate(fixture))
+      fixtureErrors.push(...validate.errors!.map(error => `${fixturePath}: ${error.instancePath || '/'} ${error.message}`))
+    if (fixture.category !== category)
+      fixtureErrors.push(`${fixturePath}: /category is "${fixture.category}", expected "${category}"`)
+  }
+}
+
+if (fixtureErrors.length > 0) {
+  console.error(fixtureErrors.join('\n'))
+  process.exit(1)
+}
+
+console.log('All fixtures match tests/fixtures.schema.json')

tests/README.md

@@ -148,17 +148,8 @@ Load each fixture file, run every entry in its `tests` array through your encode
 
 ## Validating Fixtures
 
-All fixture files should validate against [`fixtures.schema.json`](./fixtures.schema.json). Run the commands below from the repository root:
-
-```bash
-# Using ajv-cli
-npx ajv-cli validate -s tests/fixtures.schema.json -d "tests/fixtures/**/*.json"
-
-# Using check-jsonschema (Python)
-pip install check-jsonschema
-check-jsonschema --schemafile tests/fixtures.schema.json tests/fixtures/**/*.json
-```
+`pnpm test` validates every fixture against [`fixtures.schema.json`](./fixtures.schema.json) and checks that its `category` matches its directory.
 
 ## Contributing Test Cases
 
-Add your test to the matching fixture file, reference the spec section it exercises, verify the expected output against SPEC.md, and validate the file against the schema before submitting a PR. See [CONTRIBUTING.md](../CONTRIBUTING.md) for guidelines.
+Add your test to the matching fixture file, reference the spec section it exercises, verify the expected output against SPEC.md, and run `pnpm test` before submitting a PR. See [CONTRIBUTING.md](../CONTRIBUTING.md) for guidelines.