docs: settle decoder edge cases and encoder output for 4.2

CHANGELOG.md

@@ -4,6 +4,28 @@ All notable changes to the TOON specification will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). The project follows the MAJOR.MINOR versioning policy described in [VERSIONING.md](./VERSIONING.md).
 
+## [4.2] - Unreleased
+
+### Changed
+
+- Encoders: numbers use the shortest round-trip digits (§2); non-ASCII text is emitted as literal UTF-8 and `\uXXXX` escapes use lowercase hex (§7.1); strings and keys are quoted only where §7.2 or §7.3 requires it; primitive list items are quoted against the document delimiter, like object field values (§11.1).
+- Decoders – edge cases that 4.1 left open:
+  - a `"` opens a quoted span anywhere in a line (§5.2)
+  - a hyphen marks a list item only at item depth (§5.2)
+  - a line without an unquoted colon is never a header (§6)
+  - an indented first line is over-indented (§8)
+  - `: 1` decodes to the empty key; `{a,}` and `{a,{b}}` are header errors (§6, §7.4)
+  - whitespace before a nested field group is a header error (§6)
+  - a length beyond the integer range still forms a header (§6)
+  - raw control characters inside quotes decode as themselves (§7.1)
+  - a line of tabs is not blank, except under the non-strict tab leniency (§12)
+- Non-strict mode: decoders apply only the leniencies the spec names, or error (§14). New in 4.2: a jumped first line sets the scope's depth (§8), ill-formed UTF-8 may become U+FFFD (§4), and unmatched braces may fall through to key-value parsing (§6).
+- Conformance fixtures: cases for every change above; redundant cases removed, so fixture indices shift – runners that skip cases by index need new skip lists.
+
+### Compatibility
+
+Every decoder change concerns input no conforming 4.0 or 4.1 encoder emits, so no round-trip is affected. Encoders that quote more than required, print more digits than the shortest round-trip form, escape non-ASCII text, or quote list items against the active delimiter need an update; their output still decodes unchanged.
+
 ## [4.1] - 2026-07-26
 
 ### Added

tests/fixtures/decode/arrays-nested.json

@@ -84,32 +84,20 @@
       "specSection": "10"
     },
     {
-      "name": "parses objects containing arrays (including empty arrays) in list form",
+      "name": "parses a legacy [0] header as a later field of a list-item object",
       "input": "items[1]:\n  - name: Ada\n    data[0]:",
       "expected": {
-        "items": [
-          { "name": "Ada", "data": [] }
-        ]
+        "items": [{ "name": "Ada", "data": [] }]
       },
-      "specSection": "9.4"
+      "specSection": "10"
     },
     {
-      "name": "parses arrays of arrays within objects",
+      "name": "parses a list-form array as the first field of a list-item object",
       "input": "items[1]:\n  - matrix[2]:\n      - [2]: 1,2\n      - [2]: 3,4\n    name: grid",
       "expected": {
-        "items": [
-          { "matrix": [[1, 2], [3, 4]], "name": "grid" }
-        ]
-      },
-      "specSection": "9.2"
-    },
-    {
-      "name": "parses nested arrays of primitives",
-      "input": "pairs[2]:\n  - [2]: a,b\n  - [2]: c,d",
-      "expected": {
-        "pairs": [["a", "b"], ["c", "d"]]
+        "items": [{ "matrix": [[1, 2], [3, 4]], "name": "grid" }]
       },
-      "specSection": "9.2"
+      "specSection": "10"
     },
     {
       "name": "parses quoted strings and mixed lengths in nested arrays",
@@ -127,14 +115,6 @@
       },
       "specSection": "9.2"
     },
-    {
-      "name": "parses mixed-length inner arrays",
-      "input": "pairs[2]:\n  - [1]: 1\n  - [2]: 2,3",
-      "expected": {
-        "pairs": [[1], [2, 3]]
-      },
-      "specSection": "9.2"
-    },
     {
       "name": "parses root-level primitive array inline",
       "input": "[5]: x,y,\"true\",true,10",
@@ -193,25 +173,6 @@
       },
       "specSection": "9.4"
     },
-    {
-      "name": "parses arrays mixing objects and arrays",
-      "input": "items[2]:\n  - a: 1\n  - [2]: 1,2",
-      "expected": {
-        "items": [{ "a": 1 }, [1, 2]]
-      },
-      "specSection": "9.4"
-    },
-    {
-      "name": "parses quoted key with list-form array",
-      "input": "\"x-items\"[2]:\n  - id: 1\n  - id: 2",
-      "expected": {
-        "x-items": [
-          { "id": 1 },
-          { "id": 2 }
-        ]
-      },
-      "specSection": "9.4"
-    },
     {
       "name": "accepts bare bracket pair as empty inner array list item",
       "input": "items[2]:\n  - []\n  - [1]: x",
@@ -316,6 +277,17 @@
       "specSection": "14.1",
       "note": "A legacy [0] is a declared length, so its content is decoded (§14.1)",
       "minSpecVersion": "4.1"
+    },
+    {
+      "name": "parses a list item shaped like a header without a colon as a string",
+      "input": "a[1]:\n  - [2]",
+      "expected": {
+        "a": [
+          "[2]"
+        ]
+      },
+      "specSection": "6",
+      "minSpecVersion": "4.2"
     }
   ]
 }

tests/fixtures/decode/arrays-primitive.json

@@ -3,22 +3,6 @@
   "category": "decode",
   "description": "Primitive array decoding – inline arrays of strings, numbers, booleans, quoted strings",
   "tests": [
-    {
-      "name": "parses string arrays inline",
-      "input": "tags[3]: reading,gaming,coding",
-      "expected": {
-        "tags": ["reading", "gaming", "coding"]
-      },
-      "specSection": "9.1"
-    },
-    {
-      "name": "parses number arrays inline",
-      "input": "nums[3]: 1,2,3",
-      "expected": {
-        "nums": [1, 2, 3]
-      },
-      "specSection": "9.1"
-    },
     {
       "name": "parses mixed primitive arrays inline",
       "input": "data[4]: x,y,true,10",
@@ -99,14 +83,6 @@
       },
       "specSection": "9.1"
     },
-    {
-      "name": "parses quoted key containing brackets with inline array",
-      "input": "\"key[test]\"[3]: 1,2,3",
-      "expected": {
-        "key[test]": [1, 2, 3]
-      },
-      "specSection": "9.1"
-    },
     {
       "name": "parses quoted key containing a colon with inline array",
       "input": "\"a:b\"[2]: 1,2",
@@ -115,22 +91,6 @@
       },
       "specSection": "9.1"
     },
-    {
-      "name": "parses quoted key with empty array",
-      "input": "\"x-custom\"[0]:",
-      "expected": {
-        "x-custom": []
-      },
-      "specSection": "9.1"
-    },
-    {
-      "name": "parses quoted empty string key with empty array",
-      "input": "\"\"[0]:",
-      "expected": {
-        "": []
-      },
-      "specSection": "9.1"
-    },
     {
       "name": "decodes canonical empty array key: []",
       "input": "items: []",
@@ -139,22 +99,6 @@
       },
       "specSection": "9.1"
     },
-    {
-      "name": "decodes canonical empty array with quoted key",
-      "input": "\"x-custom\": []",
-      "expected": {
-        "x-custom": []
-      },
-      "specSection": "9.1"
-    },
-    {
-      "name": "decodes canonical empty array with empty-string key",
-      "input": "\"\": []",
-      "expected": {
-        "": []
-      },
-      "specSection": "9.1"
-    },
     {
       "name": "parses quoted key containing escaped quotes and brackets with inline array",
       "input": "\"has \\\"[2]\\\" in it\"[3]: 3,4,5",
@@ -208,6 +152,20 @@
         ]
       },
       "specSection": "9.3"
+    },
+    {
+      "name": "keeps a header whose length exceeds integer range in non-strict mode",
+      "input": "a[99999999999999999999]: 1",
+      "expected": {
+        "a": [
+          1
+        ]
+      },
+      "options": {
+        "strict": false
+      },
+      "specSection": "6",
+      "minSpecVersion": "4.2"
     }
   ]
 }

tests/fixtures/decode/arrays-tabular.json

@@ -25,17 +25,6 @@
       },
       "specSection": "9.3"
     },
-    {
-      "name": "parses quoted colon in tabular row as data",
-      "input": "items[2]{id,note}:\n  1,\"a:b\"\n  2,\"c:d\"",
-      "expected": {
-        "items": [
-          { "id": 1, "note": "a:b" },
-          { "id": 2, "note": "c:d" }
-        ]
-      },
-      "specSection": "9.3"
-    },
     {
       "name": "parses quoted header keys in tabular arrays",
       "input": "items[2]{\"order:id\",\"full name\"}:\n  1,Ada\n  2,Bob",
@@ -290,6 +279,41 @@
         ]
       },
       "specSection": "9.3"
+    },
+    {
+      "name": "ignores a brace inside a mid-name quote span in a field list",
+      "input": "a[1]{x\"}\"y}:\n  1",
+      "expected": {
+        "a": [
+          {
+            "x\"}\"y": 1
+          }
+        ]
+      },
+      "specSection": "6",
+      "minSpecVersion": "4.2"
+    },
+    {
+      "name": "parses a quoted empty field name",
+      "input": "[2]{\"\"}:\n  1\n  2",
+      "expected": [
+        {
+          "": 1
+        },
+        {
+          "": 2
+        }
+      ],
+      "specSection": "6",
+      "minSpecVersion": "4.2"
+    },
+    {
+      "name": "unescapes a quoted field name",
+      "input": "a[1]{\"x\\\"y\"}:\n  1",
+      "expected": {
+        "a": [{ "x\"y": 1 }]
+      },
+      "specSection": "6"
     }
   ]
 }

tests/fixtures/decode/blank-lines.json

@@ -5,7 +5,7 @@
   "tests": [
     {
       "name": "throws on blank line inside list array",
-      "input": "items[3]:\n  - a\n\n  - b\n  - c",
+      "input": "items[2]:\n  - a\n\n  - b",
       "expected": null,
       "shouldError": true,
       "options": {
@@ -55,7 +55,7 @@
     },
     {
       "name": "throws on blank line in nested list array",
-      "input": "outer[2]:\n  - inner[2]:\n    - a\n\n    - b\n  - x",
+      "input": "outer[2]:\n  - inner[2]:\n      - a\n\n      - b\n  - x",
       "expected": null,
       "shouldError": true,
       "options": {
@@ -76,7 +76,7 @@
       "specSection": "12"
     },
     {
-      "name": "accepts whitespace-only line at non-multiple indent as blank in strict mode",
+      "name": "accepts a spaces-only line at non-multiple indent as blank in strict mode",
       "input": "a: 1\n   \nb: 2",
       "expected": {
         "a": 1,
@@ -137,9 +137,9 @@
     },
     {
       "name": "ignores blank lines inside list array when strict=false",
-      "input": "items[3]:\n  - a\n\n  - b\n  - c",
+      "input": "items[2]:\n  - a\n\n  - b",
       "expected": {
-        "items": ["a", "b", "c"]
+        "items": ["a", "b"]
       },
       "options": {
         "strict": false
@@ -179,13 +179,6 @@
       "specSection": "14.2",
       "note": "The blank is after the inner scope's last row but inside the outer array's span (\u00a712)"
     },
-    {
-      "name": "throws on blank line between a list item's fields",
-      "input": "items[2]:\n  - a: 1\n\n    b: 2\n  - x",
-      "expected": null,
-      "shouldError": true,
-      "specSection": "14.2"
-    },
     {
       "name": "throws on blank line inside the last list item's fields",
       "input": "items[1]:\n  - a: 1\n\n    b: 2",
@@ -227,6 +220,27 @@
       "shouldError": true,
       "specSection": "14.2",
       "note": "The blank is after the nested object's content but inside the outer array's span (§12)"
+    },
+    {
+      "name": "throws on a tab-only line, which is not blank",
+      "input": "a: 1\n\t\nb: 2",
+      "expected": null,
+      "shouldError": true,
+      "specSection": "14.2",
+      "minSpecVersion": "4.2"
+    },
+    {
+      "name": "treats a tab-only line as blank in non-strict mode",
+      "input": "a: 1\n\t\nb: 2",
+      "expected": {
+        "a": 1,
+        "b": 2
+      },
+      "options": {
+        "strict": false
+      },
+      "specSection": "12",
+      "minSpecVersion": "4.2"
     }
   ]
 }