From ed0ca493375cb9d88813bcbd0cdd47717b012623 Mon Sep 17 00:00:00 2001 From: SaMullinsJr Date: Wed, 15 Jul 2026 12:47:01 -0400 Subject: [PATCH 1/2] harden: close all 11 cross-lineage danger-zone divergences; corpus-as-data conformance - Plain-decimal number domain in all 7 CLIs (lexical): no exponent tokens, integers limited to [-2^53, +2^53] by digit-string compare, fractions in [1e-6, 1e21). '100' accepted, '1E2' rejected. - ES-262 ToString float formatting in every lineage (shortest round-trip, plain decimal, zero -> "0"): C/OCaml %.*e+strtod loop, C++ to_chars, Rust std Display, D %.*e loop, Haskell floatToDigits. The whole fraction band [1e-6, 1e21) is now byte-identical across all seven. - Go: lone-surrogate escapes rejected (backslash-run parity), -0.0 -> "0". - OCaml: lone LOW surrogate rejected (yojson only caught the high half). - D: strict single-document parse (trailing garbage, concatenated docs, trailing comma, empty input all rejected; std.json accepted all four). - C/C++: leading UTF-8 BOM rejected. C++: over-uint64 integer tokens rejected (nlohmann routes them to number_float). - Corpus-as-data: conformance/accept.jsonl (23 pinned) + reject.jsonl (19) generated by gen_corpus.py, which refuses to pin divergent cases; verify_all_lineages.sh is corpus-driven; differential_probe.py sweeps 165 generated danger-zone cases in CI (agreement-only). - CI: actions pinned to commit SHAs; package versions -> 0.2.0; README states the precise domain (closed interval +/-2^53, plain-decimal floats, single-document rule) and prerequisites. Co-Authored-By: Claude Fable 5 --- .github/workflows/verify.yml | 23 +- README.md | 15 +- c/include/baion/canonical_json.h | 19 ++ c/src/canonical_json.c | 210 +++++++++++++- c/tests/test_canonical_json.c | 99 +++++++ c/tools/baion_canon_hash.c | 23 ++ conformance/accept.jsonl | 23 ++ conformance/differential_probe.py | 126 +++++++++ conformance/gen_corpus.py | 114 ++++++++ conformance/reject.jsonl | 19 ++ cpp/CMakeLists.txt | 2 +- cpp/include/baion/canonical_json.hpp | 25 ++ cpp/src/canonical_json.cpp | 220 ++++++++++++++- cpp/tests/test_canonical_json.cpp | 125 ++++++++- cpp/tools/baion_canon_hash.cpp | 30 +- d/source/baionstd/canonical_json.d | 364 +++++++++++++++++++++---- d/source/baionstd/types.d | 12 + d/tests/conformance_test.d | 208 +++++++++++++- d/tools/canon_hash_main.d | 31 ++- go/canonical_json.go | 169 ++++++++++++ go/canonical_json_test.go | 124 +++++++++ go/cmd/baion_canon_hash/main.go | 15 + haskell/app/Main.hs | 27 +- haskell/baionstd-public.cabal | 2 +- haskell/src/Baion/STD/CanonicalJson.hs | 162 +++++++++-- haskell/test/ConformanceTest.hs | 104 ++++++- ocaml/cli/baion_canon_hash.ml | 16 ++ ocaml/lib/canonical_json.ml | 220 ++++++++++++++- ocaml/test/conformance_test.ml | 130 +++++++++ rust/Cargo.toml | 2 +- rust/src/bin/baion_canon_hash.rs | 16 +- rust/src/canonical_json.rs | 59 ++++ rust/src/lib.rs | 1 + rust/src/num_check.rs | 270 ++++++++++++++++++ verify_all_lineages.sh | 110 ++++---- 35 files changed, 2946 insertions(+), 169 deletions(-) create mode 100644 conformance/accept.jsonl create mode 100644 conformance/differential_probe.py create mode 100644 conformance/gen_corpus.py create mode 100644 conformance/reject.jsonl create mode 100644 rust/src/num_check.rs diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 16436c9..56f5885 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -20,35 +20,35 @@ jobs: matrix: lineage: [c, cpp, rust, go, d, haskell, ocaml] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Go if: matrix.lineage == 'go' - uses: actions/setup-go@v5 + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version: '1.22' cache-dependency-path: go/go.mod - name: Set up Rust if: matrix.lineage == 'rust' - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable (2026-07-15) - name: Set up D if: matrix.lineage == 'd' - uses: dlang-community/setup-dlang@v2 + uses: dlang-community/setup-dlang@d7d85fcde7c4cd5f9a6618fce1bccc316e1e910b # v2 with: compiler: dmd-latest - name: Set up Haskell if: matrix.lineage == 'haskell' - uses: haskell-actions/setup@v2 + uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0 with: ghc-version: '9.6' cabal-version: 'latest' - name: Set up OCaml if: matrix.lineage == 'ocaml' - uses: ocaml/setup-ocaml@v3 + uses: ocaml/setup-ocaml@15d660006c1d3110d77c34b7faa3bddefe8b82f0 # v3.7.0 with: ocaml-compiler: '5.2' @@ -109,7 +109,7 @@ jobs: test -x "${{ matrix.lineage }}/bin/baion_canon_hash" printf '%s' '{"b":1,"a":[1,2]}' | "${{ matrix.lineage }}/bin/baion_canon_hash" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: cli-${{ matrix.lineage }} path: ${{ matrix.lineage }}/bin/baion_canon_hash @@ -120,9 +120,9 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: cli-* @@ -138,3 +138,8 @@ jobs: - name: Run cross-lineage verifier run: ./verify_all_lineages.sh + + # Agreement-only sweep over generated danger-zone cases: any + # accept/reject split or hash split across the seven CLIs fails. + - name: Run differential probe + run: python3 conformance/differential_probe.py diff --git a/README.md b/README.md index 9323464..50134d8 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,9 @@ Seven independent implementations of the same canonicalization contract — C, C ./verify_all_lineages.sh ``` -`build_all.sh` builds each lineage with its native toolchain, runs its test suite, and places the CLI at `/bin/baion_canon_hash` — the layout the verifier requires. `verify_all_lineages.sh` then pipes the same JSON documents into every CLI and diffs the hex, and additionally asserts that all seven **uniformly reject** inputs outside the supported domain (see below). All seven lineages must be present — a missing binary fails the run, and one byte of disagreement anywhere fails the run. Success prints `PASS: 7/7 lineages produced identical output`. +**Prerequisites** (one toolchain per lineage): a C compiler + `make`; CMake ≥ 3.20 + a C++17 compiler; Rust (`cargo`); Go ≥ 1.22; D (`dmd` + `dub`); GHC ≥ 9.6 + `cabal` (aeson ≥ 2.2 is fetched by cabal); OCaml ≥ 5.x + `dune` with `yojson`, `digestif`, `alcotest` (via opam); `python3` for the conformance tooling. A successful run ends with a 7/7 PASS table from `build_all.sh` and `PASS: 7/7 lineages agree ...` from the verifier, exit 0. + +`build_all.sh` builds each lineage with its native toolchain, runs its test suite, and places the CLI at `/bin/baion_canon_hash` — the layout the verifier requires. `verify_all_lineages.sh` then feeds every vector in the conformance corpus (`conformance/accept.jsonl` + `conformance/reject.jsonl`) to every CLI: accept vectors must hash to the corpus-pinned SHA-256 in all seven lineages, and reject vectors must be **uniformly refused** (see below). `conformance/differential_probe.py` additionally sweeps generated danger-zone cases (number bands, escape forms, document framing) and fails on any disagreement; both run in CI. All seven lineages must be present — a missing binary fails the run, and one byte of disagreement anywhere fails the run. Success prints `PASS: 7/7 lineages produced identical output`. ## Why this exists @@ -22,15 +24,18 @@ The interesting engineering is in the edge cases: key ordering, number formattin Cross-lineage byte-identity is enforced and tested for: - objects (member names must be **unique** — see below), arrays, strings (full UTF-8, including multi-byte and escaped control characters **except U+0000**), booleans, null -- integers within the IEEE-754 exact range (±2⁵³) -- floats whose canonical form is pinned by the conformance vectors (including integer-valued floats, which serialize without a trailing `.0` per RFC 8785 §3.2.2.3 — `1.0` canonicalizes to `1` in every lineage, enforced by the verifier) +- integers in the closed interval [−2⁵³, +2⁵³] (i.e. |n| ≤ 9007199254740992). Every integer in this interval is exactly representable in IEEE-754 binary64. Note this is one wider than JavaScript's `MAX_SAFE_INTEGER` (2⁵³ − 1): ±2⁵³ itself is admitted because it is exact and unambiguous *within this domain* — the neighboring value 2⁵³ + 1 (the first integer that would silently round to it) is rejected, so no two accepted integer tokens can collide +- floats in the **plain-decimal domain**: zero, or magnitude in [10⁻⁶, 10²¹), written without exponent notation; the canonical form is ECMAScript `ToString` (shortest round-trip digits, plain decimal) per RFC 8785 §3.2.2.3 — `1.0` → `1`, `-0.0` → `0`, `0.1` → `0.1`, in every lineage, enforced by the pinned corpus -**Uniformly rejected:** two input classes are refused with a nonzero exit by all seven CLIs, and the verifier asserts the rejection is uniform: +**Uniformly rejected:** these input classes are refused with a nonzero exit by all seven CLIs, and the verifier asserts the rejection is uniform: - *Strings containing U+0000* (as the escape `\u0000` or a raw NUL byte). One lineage cannot represent embedded NUL losslessly, and accepting it anywhere would allow silent canonicalization collisions. A literal backslash followed by the text `u0000` (JSON `\\u0000`) is not a NUL and canonicalizes normally. - *Objects with duplicate member names*, at any nesting depth. RFC 8259 leaves duplicate-name behavior undefined, and the seven JSON ecosystems genuinely diverge (keep-first, keep-last, keep-both) — so object member names must be unique. Duplicates are detected on the *decoded* name: `{"a":1,"\u0061":2}` is rejected because `\u0061` decodes to `a`. +- *Numbers outside the plain-decimal domain*: exponent notation (`1e2` is rejected even though the value is in range — spell it `100`), integers beyond ±2⁵³, and fractions below 10⁻⁶ or at/above 10²¹. Seven number formatters genuinely disagree in exponent territory; the supported domain is exactly where byte-identity is provable, and inside it the output is normalized rather than excluded. +- *Unpaired surrogate escapes* (a `\ud800`–`\udbff` escape not immediately followed by a low half, or a lone `\udc00`–`\udfff`): not Unicode scalar values, and ecosystems differ on replacement behavior. A literal backslash followed by surrogate text (`\\ud800`) is ordinary content. +- *Anything other than exactly one JSON document*: a leading UTF-8 BOM, trailing non-whitespace, concatenated documents, trailing commas, or empty input. -**Known exclusions** (documented honestly because they are where seven ecosystems genuinely differ): number formatting outside the pinned vectors — very large integers beyond 2⁵³, negative zero, and scientific-notation thresholds — is not yet normalized across all seven lineages and must not be relied on. Non-finite numbers (NaN, ±Inf) are not valid JSON and are rejected or nulled per lineage test suites. If your data stays in the supported domain, the byte-identity guarantee holds; the conformance fixture is the authoritative definition of that domain. +**Known exclusions:** non-finite numbers (NaN, ±Inf) are not valid JSON and are rejected or nulled per lineage test suites. The conformance corpus (`conformance/accept.jsonl`, pinned hashes; `conformance/reject.jsonl`, uniform rejections; regenerated by `conformance/gen_corpus.py`, which refuses to pin any case the seven CLIs disagree on) is the authoritative definition of the supported domain. If your data stays in the supported domain, the byte-identity guarantee holds. ## Layout diff --git a/c/include/baion/canonical_json.h b/c/include/baion/canonical_json.h index a603909..efcb7ed 100644 --- a/c/include/baion/canonical_json.h +++ b/c/include/baion/canonical_json.h @@ -22,4 +22,23 @@ int baion_reject_u0000(const char* input, size_t len); * escaped-form duplicates. */ int baion_reject_duplicate_keys(const cJSON* root); +/* Pre-parse scan of raw JSON input bytes: returns BAION_OK if the first + * bytes are NOT a UTF-8 byte-order mark (EF BB BF), else BAION_ERR_PARSE. + * Must run BEFORE cJSON parsing — cJSON silently skips a leading BOM, so a + * BOM-prefixed document and its BOM-free twin would hash identically while + * being byte-distinct on the wire. */ +int baion_reject_bom(const char* input, size_t len); + +/* Pre-parse LEXICAL scan of raw JSON number tokens: returns BAION_OK if every + * number token in the input is inside the plain-decimal domain, else + * BAION_ERR_PARSE. Must run BEFORE cJSON parsing — cJSON collapses "100" and + * "1e2" onto the same double, so only the raw token spelling can distinguish + * them. Out of domain: any exponent notation (e/E); integer tokens (no '.') + * whose magnitude exceeds 2^53 (digit-string compare, never via double); + * fraction tokens whose value v has (v != 0 && |v| < 1e-6) or |v| >= 1e21. + * Precondition: input[len] == '\0' (the fraction check hands the token + * suffix to strtod, which stops at the token's non-number delimiter or at + * that terminator). */ +int baion_reject_number_domain(const char* input, size_t len); + #endif /* BAION_CANONICAL_JSON_H */ diff --git a/c/src/canonical_json.c b/c/src/canonical_json.c index 9f5b9c7..efe6dcc 100644 --- a/c/src/canonical_json.c +++ b/c/src/canonical_json.c @@ -194,10 +194,91 @@ static void write_number(strbuf_t* sb, const cJSON* item) } else { - /* Float: use %.17g then strip trailing zeros after decimal point */ - char buf[64]; - snprintf(buf, sizeof(buf), "%.17g", d); - strbuf_appends(sb, buf); + /* CROSS-LINEAGE CONTRACT: non-integer floats serialize as the + * SHORTEST decimal string that roundtrips to the same double + * (RFC 8785 §3.2.2.3 / ECMA-262 §7.1.12.1), reassembled WITHOUT + * exponent notation. The number-domain gate guarantees the value + * is 0 or |v| in [1e-6, 1e21), which is exactly the range where + * ECMA-262 ToString never takes its exponent branch — so plain + * positional layout is the canonical spelling. 17-digit %.17g + * here previously printed 0.1 as 0.10000000000000001, diverging + * from every RFC 8785 lineage. */ + + /* Shortest digits: smallest precision whose %e output parses back + * to the identical double. Locale-sensitive (%e / strtod use the + * locale decimal point): this code assumes the default "C" locale; + * if the process ever calls setlocale() with a comma-decimal + * locale, the '.' scanning below breaks. */ + char sci[64]; + for (int prec = 1; prec <= 17; prec++) + { + snprintf(sci, sizeof(sci), "%.*e", prec - 1, d); + if (strtod(sci, NULL) == d) + break; + } + + /* Pull apart [-]d[.ddd]e±XX into digit string D and n = exp10 + 1 + * (count of digits before the decimal point in positional form). */ + const char* s = sci; + int neg = 0; + if (*s == '-') + { + neg = 1; + s++; + } + char digits[32]; + int dl = 0; + digits[dl++] = *s++; + if (*s == '.') + { + s++; + while (*s != 'e' && *s != 'E') + digits[dl++] = *s++; + } + s++; /* skip 'e'; strtol consumes the +/- sign of the exponent */ + int n = (int)strtol(s, NULL, 10) + 1; + + char out[64]; + int o = 0; + if (neg) + out[o++] = '-'; + if (dl <= n && n <= 21) + { + /* Integer-valued but too large for the %lld branch above + * (|v| >= 1e15): all digits then zero-padding, no dot. */ + memcpy(out + o, digits, (size_t)dl); + o += dl; + for (int k = 0; k < n - dl; k++) + out[o++] = '0'; + } + else if (0 < n && n <= dl) + { + memcpy(out + o, digits, (size_t)n); + o += n; + out[o++] = '.'; + memcpy(out + o, digits + n, (size_t)(dl - n)); + o += dl - n; + } + else if (-5 <= n && n <= 0) + { + out[o++] = '0'; + out[o++] = '.'; + for (int k = 0; k < -n; k++) + out[o++] = '0'; + memcpy(out + o, digits, (size_t)dl); + o += dl; + } + else + { + /* Only reachable when a caller bypassed the number-domain gate + * (programmatic tree with |v| >= 1e21 or |v| < 1e-6): emit a + * best-effort spelling rather than corrupt memory. NON-CANONICAL. */ + snprintf(out, sizeof(out), "%.17g", d); + strbuf_appends(sb, out); + return; + } + out[o] = '\0'; + strbuf_appends(sb, out); } } @@ -301,6 +382,127 @@ int baion_reject_duplicate_keys(const cJSON* root) return BAION_OK; } +int baion_reject_bom(const char* input, size_t len) +{ + /* cJSON silently skips a leading UTF-8 BOM, so "\xEF\xBB\xBF{...}" and + * "{...}" would collapse onto one canonical form despite being distinct + * byte streams — same collision class as U+0000 above. */ + if (len >= 3 && (unsigned char)input[0] == 0xEF && (unsigned char)input[1] == 0xBB + && (unsigned char)input[2] == 0xBF) + return BAION_ERR_PARSE; + return BAION_OK; +} + +/* CROSS-LINEAGE CONTRACT: the plain-decimal number domain is enforced + * LEXICALLY over the raw token in every lineage — "100" is in-domain while + * "1e2" is not, even though both parse to the same double. A post-parse + * check cannot make that distinction, so this scan must run before cJSON. */ +int baion_reject_number_domain(const char* input, size_t len) +{ + /* 2^53: largest magnitude every lineage's double represents exactly for + * ALL integers up to it. Compared as a digit string — a double compare + * would round 9007199254740993 down to 2^53 and wave it through. */ + static const char max_safe[] = "9007199254740992"; + const size_t max_safe_len = sizeof(max_safe) - 1; + + size_t i = 0; + while (i < len) + { + char c = input[i]; + + if (c == '"') + { + /* Skip string literals so digits inside text (e.g. "1e400" as a + * VALUE) are never mistaken for number tokens. Same escape rule + * as baion_reject_u0000: a backslash neutralizes the next char, + * so an escaped quote cannot close the string. */ + i++; + while (i < len && input[i] != '"') + { + if (input[i] == '\\' && i + 1 < len) + i++; + i++; + } + i++; /* closing quote */ + continue; + } + + if (c == '-' || (c >= '0' && c <= '9')) + { + /* Number token: consume the JSON number grammar's alphabet. + * '+'/'-' are only legal at token start or inside an exponent; + * anywhere else they end the token (invalid JSON — the parser + * rejects it after this scan). */ + size_t start = i; + int has_dot = 0; + int has_exp = 0; + while (i < len) + { + char t = input[i]; + if (t >= '0' && t <= '9') + { + /* digit: always part of the token */ + } + else if (t == '.') + has_dot = 1; + else if (t == 'e' || t == 'E') + has_exp = 1; + else if ((t == '+' || t == '-') && (i == start || has_exp)) + { + /* sign: leading minus or exponent sign only */ + } + else + break; + i++; + } + + /* Exponent notation is outside the plain-decimal domain even + * when the VALUE is in range: "1E2" and "100" must not both + * canonicalize (the raw spellings differ, the doubles do not). */ + if (has_exp) + return BAION_ERR_PARSE; + + if (!has_dot) + { + /* Integer token: digit-string magnitude compare vs 2^53. + * Sign and leading zeros carry no magnitude — strip them, + * then longer wins, else lexicographic decides. */ + const char* p = input + start; + size_t tlen = i - start; + if (tlen > 0 && *p == '-') + { + p++; + tlen--; + } + while (tlen > 1 && *p == '0') + { + p++; + tlen--; + } + if (tlen > max_safe_len + || (tlen == max_safe_len && strncmp(p, max_safe, max_safe_len) > 0)) + return BAION_ERR_PARSE; + } + else + { + /* Fraction token: the writer's no-exponent reassembly is + * only the canonical ECMA-262 spelling for 0 or |v| in + * [1e-6, 1e21) — outside that, ToString would switch to + * exponent form, so the value is out of domain. strtod + * stops at the same delimiter this scan stopped at (or the + * caller-guaranteed NUL at input[len]). */ + double v = strtod(input + start, NULL); + if ((v != 0.0 && fabs(v) < 1e-6) || fabs(v) >= 1e21) + return BAION_ERR_PARSE; + } + continue; + } + + i++; + } + return BAION_OK; +} + char* baion_canonicalize_json(const cJSON* value) { strbuf_t sb; diff --git a/c/tests/test_canonical_json.c b/c/tests/test_canonical_json.c index 2d5c922..bd0882f 100644 --- a/c/tests/test_canonical_json.c +++ b/c/tests/test_canonical_json.c @@ -174,6 +174,102 @@ static void test_duplicate_key_rejection(void) ASSERT_INT_EQ(dup_check("{\"b\":1,\"a\":[1,2]}"), BAION_OK); } +static void test_float_shortest_roundtrip(void) +{ + /* Each case: shortest decimal string that roundtrips to the same double, + * positional layout (no exponent), per RFC 8785 / ECMA-262. The old + * %.17g path printed 0.1 as "0.10000000000000001", diverging from every + * other lineage. */ + static const struct + { + double v; + const char* expect; + } cases[] = { + {0.1, "0.1"}, + {123.456, "123.456"}, + {0.5, "0.5"}, + {0.000001, "0.000001"}, /* n = -5 boundary: "0." + 5 zeros + D */ + {1e16, "10000000000000000"}, /* dl < n: zero-padded, no dot */ + {9007199254740992.0, "9007199254740992"}, /* 2^53: above %lld branch cutoff */ + {-2.5, "-2.5"}, + {1.0, "1"}, /* integer-valued keeps no-trailing-".0" behavior */ + {-0.0, "0"}, /* zero (incl. negative zero) always emits exactly "0" */ + }; + + for (size_t k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) + { + cJSON* n = cJSON_CreateNumber(cases[k].v); + char* json = baion_canonicalize_json(n); + ASSERT_STR_EQ(json, cases[k].expect); + free(json); + cJSON_Delete(n); + } +} + +static void test_number_domain_rejection(void) +{ + /* Exponent notation: out of domain even when the value is in range — + * the scan is LEXICAL, "1e2" and "100" must not both canonicalize. */ + const char* exp_lower = "{\"x\":1e2}"; + ASSERT_INT_EQ(baion_reject_number_domain(exp_lower, strlen(exp_lower)), BAION_ERR_PARSE); + const char* exp_upper = "{\"x\":1E5}"; + ASSERT_INT_EQ(baion_reject_number_domain(exp_upper, strlen(exp_upper)), BAION_ERR_PARSE); + const char* exp_neg = "{\"x\":1e-7}"; + ASSERT_INT_EQ(baion_reject_number_domain(exp_neg, strlen(exp_neg)), BAION_ERR_PARSE); + const char* exp_huge = "{\"x\":1e400}"; + ASSERT_INT_EQ(baion_reject_number_domain(exp_huge, strlen(exp_huge)), BAION_ERR_PARSE); + + /* Same value spelled plainly: allowed */ + const char* plain = "{\"x\":100}"; + ASSERT_INT_EQ(baion_reject_number_domain(plain, strlen(plain)), BAION_OK); + + /* Integer magnitude: 2^53 itself stays in-domain; 2^53 + 1 is rejected + * in both signs (a double compare would round it down and miss it). */ + const char* at_limit = "{\"x\":9007199254740992}"; + ASSERT_INT_EQ(baion_reject_number_domain(at_limit, strlen(at_limit)), BAION_OK); + const char* over_limit = "{\"x\":9007199254740993}"; + ASSERT_INT_EQ(baion_reject_number_domain(over_limit, strlen(over_limit)), BAION_ERR_PARSE); + const char* neg_over = "{\"x\":-9007199254740993}"; + ASSERT_INT_EQ(baion_reject_number_domain(neg_over, strlen(neg_over)), BAION_ERR_PARSE); + + /* Longer digit string: rejected by length before any lexicographic step */ + const char* long_int = "{\"x\":100000000000000000000}"; + ASSERT_INT_EQ(baion_reject_number_domain(long_int, strlen(long_int)), BAION_ERR_PARSE); + + /* Fraction magnitude: |v| must be 0 or in [1e-6, 1e21) */ + const char* tiny = "{\"x\":0.0000001}"; + ASSERT_INT_EQ(baion_reject_number_domain(tiny, strlen(tiny)), BAION_ERR_PARSE); + const char* micro = "{\"x\":0.000001}"; + ASSERT_INT_EQ(baion_reject_number_domain(micro, strlen(micro)), BAION_OK); + const char* huge_frac = "{\"x\":1000000000000000000000.5}"; + ASSERT_INT_EQ(baion_reject_number_domain(huge_frac, strlen(huge_frac)), BAION_ERR_PARSE); + const char* zero_frac = "{\"x\":0.0}"; + ASSERT_INT_EQ(baion_reject_number_domain(zero_frac, strlen(zero_frac)), BAION_OK); + + /* Number-shaped text inside string literals is NOT a number token — + * including behind an escaped quote, which must not close the string. */ + const char* in_string = "{\"x\":\"1e400\"}"; + ASSERT_INT_EQ(baion_reject_number_domain(in_string, strlen(in_string)), BAION_OK); + const char* esc_quote = "{\"a\\\"e\":\"9E99\",\"x\":2}"; + ASSERT_INT_EQ(baion_reject_number_domain(esc_quote, strlen(esc_quote)), BAION_OK); +} + +static void test_bom_rejection(void) +{ + /* Leading UTF-8 BOM: cJSON would silently skip it, collapsing the + * BOM-prefixed and BOM-free documents onto one hash — reject. */ + const char* bom_doc = "\xEF\xBB\xBF{\"a\":1}"; + ASSERT_INT_EQ(baion_reject_bom(bom_doc, strlen(bom_doc)), BAION_ERR_PARSE); + + /* Same document without the BOM: allowed */ + const char* clean = "{\"a\":1}"; + ASSERT_INT_EQ(baion_reject_bom(clean, strlen(clean)), BAION_OK); + + /* BOM bytes NOT at the very start are just string content: allowed */ + const char* interior = "{\"a\":\"\xEF\xBB\xBF\"}"; + ASSERT_INT_EQ(baion_reject_bom(interior, strlen(interior)), BAION_OK); +} + int main(void) { printf("test_canonical_json:\n"); @@ -185,6 +281,9 @@ int main(void) RUN_TEST(test_booleans); RUN_TEST(test_u0000_rejection); RUN_TEST(test_duplicate_key_rejection); + RUN_TEST(test_float_shortest_roundtrip); + RUN_TEST(test_number_domain_rejection); + RUN_TEST(test_bom_rejection); TEST_SUMMARY(); return _test_failed; } diff --git a/c/tools/baion_canon_hash.c b/c/tools/baion_canon_hash.c index 471cd44..d48af1e 100644 --- a/c/tools/baion_canon_hash.c +++ b/c/tools/baion_canon_hash.c @@ -54,6 +54,16 @@ int main(void) return -BAION_ERR_PARSE; } + /* Pre-parse BOM rejection (library-level scan): cJSON silently skips a + leading UTF-8 BOM, so a BOM-prefixed document would hash identically + to its BOM-free twin. Reviewer contract: reject with exit 1. */ + if (baion_reject_bom(input, len) != BAION_OK) + { + fprintf(stderr, "baion_canon_hash: leading UTF-8 BOM in input is unsupported\n"); + free(input); + return 1; + } + /* Pre-parse U+0000 rejection (library-level scan): cJSON decodes the u0000 escape into a NUL byte that truncates the C string, so distinct documents would collapse onto one canonical form. Reviewer contract: @@ -65,6 +75,19 @@ int main(void) return 1; } + /* Pre-parse number-domain rejection (library-level LEXICAL scan): cJSON + collapses "100" and "1e2" onto the same double, so only the raw token + spelling can enforce the plain-decimal domain (no exponent notation, + integers within 2^53, fractions in [1e-6, 1e21)). Reviewer contract: + reject with exit 1. */ + if (baion_reject_number_domain(input, len) != BAION_OK) + { + fprintf(stderr, + "baion_canon_hash: number outside plain-decimal domain is unsupported\n"); + free(input); + return 1; + } + /* require_null_terminated=1 rejects trailing garbage after the document (whitespace is allowed by cJSON's skip). */ const char* parse_end = NULL; diff --git a/conformance/accept.jsonl b/conformance/accept.jsonl new file mode 100644 index 0000000..1588905 --- /dev/null +++ b/conformance/accept.jsonl @@ -0,0 +1,23 @@ +{"name": "key_sort_basic", "input": "{\"b\":1,\"a\":[1,2]}", "sha256": "94a786c3662bc7beeb598efa7d8cb58d7bea25d6c275ea9785a0230ff1f8c2ba"} +{"name": "key_sort_unicode", "input": "{\"z\":1,\"a\":\"\u00e9\"}", "sha256": "fb64e573f7cde5b7efeda52ffc4bdd57572055b0b7e64a70172606c82c6c7eac"} +{"name": "nested_mixed", "input": "{\"nested\":{\"y\":[true,false,null],\"x\":0.5},\"empty\":{},\"arr\":[]}", "sha256": "0a9f877d8a670ed579d21f5b7e32d870cde347fc0e6450d9f284724217f025da"} +{"name": "unicode_keys", "input": "{\"\u00e9\":1,\"e\":2,\"z\u00df\":\"stra\u00dfe\"}", "sha256": "f9eaba80c1a925d133b1adfe9ae4e2ffc486048dc216131f40a1152ef0ddc665"} +{"name": "escape_zoo", "input": "{\"escapes\":\"line\\nbreak\\ttab \\\"quoted\\\" back\\\\slash\"}", "sha256": "fb152eccf9225e2728e71bdcdaa0781912053ee5bce83d7195908f929a1f7cef"} +{"name": "max_safe_int", "input": "{\"max_safe\":9007199254740992,\"neg\":-42,\"empty\":\"\"}", "sha256": "b8913d02715acee3830bff9f7aa690cc57b9abce0d003292e87e8b0b57f142ca"} +{"name": "deep_structure", "input": "{\"deep\":{\"a\":{\"b\":{\"c\":[1,{\"d\":[]}]}}}}", "sha256": "3fc8cc84e97caacd0c18f216100e13f3ae35c4f5569b0a193b706e9b217c4362"} +{"name": "float_int_normalizes", "input": "{\"x\":1.0}", "sha256": "5041bf1f713df204784353e82f6a4a535931cb64f1f4b4a5aeaffcb720918b22"} +{"name": "bare_float_int", "input": "1.0", "sha256": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"} +{"name": "literal_backslash_u0000", "input": "{\"x\":\"a\\\\u0000b\"}", "sha256": "28b57382737c1f94b8d05d8cfa2978d0c28b377a661afe339c4654e6550b6a0b"} +{"name": "near_duplicate_keys", "input": "{\"aa\":1,\"ab\":2}", "sha256": "33528cbdd6efd8bbdc3fb2681a33fda1da9390813389d4afab0a2f6247088fdc"} +{"name": "neg_zero_int", "input": "{\"x\":-0}", "sha256": "5bff452c5ed93f2e87a23984db5a15050c6477335fdec955b70063bb2d692bf1"} +{"name": "neg_zero_float", "input": "{\"x\":-0.0}", "sha256": "5bff452c5ed93f2e87a23984db5a15050c6477335fdec955b70063bb2d692bf1"} +{"name": "plain_decimal", "input": "{\"x\":0.1}", "sha256": "2b018c1708c1cc2f4a8ed7f085c2cc748153117b173f7c076f7d5813bbb50f21"} +{"name": "half", "input": "{\"x\":0.5}", "sha256": "ad0233224b23c9622ebd084088f2c1647737a4cabad828e4349bbed0d9b42e11"} +{"name": "surrogate_pair_emoji", "input": "{\"x\":\"\ud83d\ude00\"}", "sha256": "c10cb8a0c573e0eafbed00d33a65f0a83da0cb03445908aed65f7deecc63019e"} +{"name": "escaped_solidus", "input": "{\"x\":\"\\/\"}", "sha256": "4e1f6444c749458a35a133c1be1fbaee3eb00ee295ed3c0f2b28d84e486087ae"} +{"name": "noncharacter_ok", "input": "{\"x\":\"\uffff\"}", "sha256": "a4ad7293647eb9d18da9c11151fa5ab9404ad22ed0db34e68ab73555ecf81559"} +{"name": "bare_string", "input": "\"hi\"", "sha256": "b49177e05868b7af8e82a644c1ce20e521af46497adeaffe861d294d9b4bb75e"} +{"name": "bare_true", "input": "true", "sha256": "b5bea41b6c623f7c09f1bf24dcae58ebab3c0cdd90ad966bc43a45b44867e12b"} +{"name": "deep_nest_60", "input": "{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}", "sha256": "8f86b55fdc801b4c73916389d1f83f057412a6e333132529f2fb44bc192956d0"} +{"name": "same_key_sibling_objects", "input": "[{\"k\":1},{\"k\":2}]", "sha256": "98fcf287e1991c1602a189793606501715f8ae194db5dcaaf6515ed29937c20d"} +{"name": "micro_boundary", "input": "{\"x\":0.000001}", "sha256": "2d6412ab0155bb89d63b73dbf83334b26b39d03e8c832cc253c6a6caceba9733"} diff --git a/conformance/differential_probe.py b/conformance/differential_probe.py new file mode 100644 index 0000000..912f9dc --- /dev/null +++ b/conformance/differential_probe.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# BAION STD differential probe — Observer (agreement-only fuzz over danger zones) +# Spec: repo README "Supported JSON domain"; corpus-as-data refactor (harden-v0.2.0). +# +# WHY this exists: the pinned corpus (accept.jsonl/reject.jsonl) proves the +# lineages agree on KNOWN vectors; this probe generates a deterministic sweep +# of the danger zones (number bands, escape forms, structure edges) and fails +# if the seven CLIs disagree on ANY case — either a split accept/reject +# decision or two different hashes. No pinned hashes here: agreement is the +# only assertion, so new divergences surface as failures instead of silently +# waiting for a reviewer to find them. Deterministic by construction (no +# randomness) so CI failures are reproducible. +# +# Inputs are built as Python strings and encoded to UTF-8; escape-sensitive +# cases are assembled from chr()/concatenation so no editor can decode them. +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +LINEAGES = ["c", "cpp", "rust", "go", "d", "haskell", "ocaml"] + +BS = chr(0x5C) # backslash, kept out of literals so editors can't decode escapes + + +def plain_decimal(digits: str, exp10: int) -> str: + """Write digits × 10^exp10 as a plain-decimal token (never exponent form).""" + if exp10 >= 0: + return digits + "0" * exp10 + if -exp10 < len(digits): + return digits[:exp10] + "." + digits[exp10:] + return "0." + "0" * (-exp10 - len(digits)) + digits + + +def build_cases(): + cases = [] # (name, input-text) + + # Integer tokens across magnitudes (in-domain and beyond ±2^53). + for v in ["0", "-0", "1", "-1", "42", "-42", "9007199254740992", + "-9007199254740992", "9007199254740993", "-9007199254740993", + "99999999999999999999999999", "-99999999999999999999999999"]: + cases.append((f"int_{v}", '{"x":' + v + "}")) + for e in range(0, 24): + cases.append((f"int_pow10_{e}", '{"x":1' + "0" * e + "}")) + + # Fraction tokens: d × 10^e written plainly, sweeping the whole band + # including out-of-domain edges (which must uniformly reject). + for digits in ["1", "15", "123456789"]: + for e in range(-12, 23 - len("123456789")): + tok = plain_decimal(digits, e) + if "." not in tok: + tok += ".0" # force the fraction lexical class + cases.append((f"frac_{digits}e{e}", '{"x":' + tok + "}")) + for v in ["-0.0", "0.5", "-0.375", "0.1", "123.456", "3.141592653589793", + "0.000001", "0.0000001", "1.7976931348623157", "2.220446049250313"]: + cases.append((f"frac_{v}", '{"x":' + v + "}")) + + # Exponent spellings — outside the plain-decimal domain, must reject. + for v in ["1e0", "1E0", "1e2", "1e-2", "1e-7", "1e21", "1e400", "-1e400", + "1.5e3", "2E-6"]: + cases.append((f"exp_{v}", '{"x":' + v + "}")) + + # Strings: escape zoo, NUL, surrogates, multibyte. + cases.append(("str_escape_zoo", + '{"s":"a' + BS + 'n b' + BS + 't c' + BS + '" d' + BS + BS + ' e' + BS + '/"}')) + cases.append(("str_nul_escape", '{"s":"a' + BS + 'u0000b"}')) # reject + cases.append(("str_literal_backslash_u0000", '{"s":"a' + BS + BS + 'u0000b"}')) + cases.append(("str_lone_high_surrogate", '{"s":"' + BS + 'ud800"}')) # reject + cases.append(("str_lone_low_surrogate", '{"s":"' + BS + 'udc00"}')) # reject + cases.append(("str_surrogate_pair_escape", '{"s":"' + BS + 'ud83d' + BS + 'ude00"}')) + cases.append(("str_raw_emoji", '{"s":"\U0001F600"}')) + cases.append(("str_escaped_a_vs_raw_key", '{"a":1,"' + BS + 'u0061":2}')) # reject (dup) + cases.append(("str_noncharacter", '{"s":"￿"}')) + cases.append(("str_control_escapes", '{"s":"' + BS + 'u0001' + BS + 'u001f"}')) + cases.append(("str_long", '{"s":"' + "xy" * 512 + '"}')) + + # Structure: duplicates, depth, document framing. + cases.append(("dup_toplevel", '{"a":1,"a":2}')) # reject + cases.append(("dup_nested", '{"x":{"b":1,"b":2}}')) # reject + cases.append(("dup_in_array", '[{"k":1,"k":2}]')) # reject + cases.append(("sibling_same_key", '[{"k":1},{"k":2}]')) + cases.append(("deep_nest_60", ('{"a":' * 60) + "1" + ("}" * 60))) + cases.append(("bom_prefix", chr(0xFEFF) + '{"a":1}')) # reject + cases.append(("trailing_garbage", '{"a":1} x')) # reject + cases.append(("two_documents", '{"a":1}{"b":2}')) # reject + cases.append(("trailing_comma", '{"a":1,}')) # reject + cases.append(("empty_input", "")) # reject + cases.append(("surrounding_whitespace", ' {"a":1} ')) + cases.append(("newline_framing", '\n{"a":1}\n')) + for bare in ['"hi"', "true", "false", "null", "1", "1.5", "[]", "{}"]: + cases.append((f"bare_{bare[:6]}", bare)) + return cases + + +def run_cli(lineage: str, payload: bytes): + cli = ROOT / lineage / "bin" / "baion_canon_hash" + p = subprocess.run([str(cli)], input=payload, capture_output=True) + return p.returncode, p.stdout.decode(errors="replace").strip() + + +def main() -> int: + cases = build_cases() + divergent = [] + for name, text in cases: + payload = text.encode("utf-8") + results = {L: run_cli(L, payload) for L in LINEAGES} + accepts = {L for L, (rc, _) in results.items() if rc == 0} + if accepts and accepts != set(LINEAGES): + divergent.append((name, "accept/reject split", results)) + continue + hashes = {h for rc, h in results.values() if rc == 0} + if len(hashes) > 1: + divergent.append((name, "hash split", results)) + + print(f"probed {len(cases)} cases across {len(LINEAGES)} lineages: " + f"{len(divergent)} divergent") + for name, kind, results in divergent: + print(f"DIVERGE ({kind}): {name}") + for L, (rc, h) in results.items(): + print(f" {L:8s} {'REJECT' if rc else h}") + return 1 if divergent else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/gen_corpus.py b/conformance/gen_corpus.py new file mode 100644 index 0000000..1066a4a --- /dev/null +++ b/conformance/gen_corpus.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# BAION STD conformance corpus generator — Make (regenerate accept.jsonl/reject.jsonl) +# Spec: repo README "Supported JSON domain"; corpus-as-data refactor (harden-v0.2.0). +# +# WHY this exists: vectors used to live inline in verify_all_lineages.sh and in seven +# per-lineage test files; every new danger zone meant editing eight places. The corpus +# is now the single source of truth — this script pins each accept case's SHA-256 by +# running ALL SEVEN current CLIs and refusing to pin anything they disagree on, so a +# stale or divergent binary can never mint a wrong expected hash. +# +# Inputs are stored as JSON-escaped strings; consumers must decode the JSON string and +# feed the resulting bytes (UTF-8) to the CLI. Escape-sensitive vectors (the NUL escape, the u0061-spelled key, ud800, the BOM) survive this encoding losslessly, which raw shell arrays did not. +import json +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +LINEAGES = ["c", "cpp", "rust", "go", "d", "haskell", "ocaml"] + +BS = chr(0x5C) # backslash, kept out of literals so editors can't decode escapes +NUL_ESC = BS + "u0000" +A_ESC = BS + "u0061" +LONE_SURROGATE = BS + "ud800" +BOM = chr(0xFEFF) + +# name, input-text. Accept cases get hashes pinned at generation time. +ACCEPT = [ + ("key_sort_basic", '{"b":1,"a":[1,2]}'), + ("key_sort_unicode", '{"z":1,"a":"é"}'), + ("nested_mixed", '{"nested":{"y":[true,false,null],"x":0.5},"empty":{},"arr":[]}'), + ("unicode_keys", '{"é":1,"e":2,"zß":"straße"}'), + ("escape_zoo", '{"escapes":"line' + BS + 'nbreak' + BS + 'ttab ' + BS + '"quoted' + BS + '" back' + BS + BS + 'slash"}'), + ("max_safe_int", '{"max_safe":9007199254740992,"neg":-42,"empty":""}'), + ("deep_structure", '{"deep":{"a":{"b":{"c":[1,{"d":[]}]}}}}'), + ("float_int_normalizes", '{"x":1.0}'), + ("bare_float_int", "1.0"), + ("literal_backslash_u0000", '{"x":"a' + BS + BS + 'u0000b"}'), + ("near_duplicate_keys", '{"aa":1,"ab":2}'), + ("neg_zero_int", '{"x":-0}'), + ("neg_zero_float", '{"x":-0.0}'), + ("plain_decimal", '{"x":0.1}'), + ("half", '{"x":0.5}'), + ("surrogate_pair_emoji", '{"x":"\U0001F600"}'), + ("escaped_solidus", '{"x":"' + BS + '/"}'), + ("noncharacter_ok", '{"x":"￿"}'), + ("bare_string", '"hi"'), + ("bare_true", "true"), + ("deep_nest_60", ('{"a":' * 60) + "1" + ("}" * 60)), + ("same_key_sibling_objects", '[{"k":1},{"k":2}]'), + ("micro_boundary", '{"x":0.000001}'), +] + +# Uniform-rejection contract. reason is documentation for humans + remediation maps. +REJECT = [ + ("nul_in_value", '{"x":"a' + NUL_ESC + 'b"}', "U+0000 not representable losslessly in all lineages"), + ("nul_in_key", '{"a' + NUL_ESC + '":1}', "U+0000 not representable losslessly in all lineages"), + ("dup_key_toplevel", '{"a":1,"a":2}', "RFC 8259 duplicate-name behavior undefined; three-way divergence observed"), + ("dup_key_nested", '{"x":{"b":1,"b":2}}', "duplicate detection required at any depth"), + ("dup_key_escaped", '{"a":1,"' + A_ESC + '":2}', "duplicates compare on the DECODED name"), + ("dup_key_in_array", '[{"k":1,"k":2}]', "duplicate detection inside objects nested in arrays"), + ("exponent_lower", '{"x":1e21}', "exponent tokens outside plain-decimal domain"), + ("exponent_upper", '{"x":1E5}', "exponent tokens outside plain-decimal domain (case-insensitive)"), + ("exponent_tiny", '{"x":1e-7}', "five-way float-format divergence observed"), + ("exponent_overflow", '{"x":1e400}', "overflows double; three accept/four reject divergence observed"), + ("int_beyond_2_53", '{"x":9007199254740993}', "beyond IEEE-754 exact-integer range"), + ("neg_int_beyond_2_53", '{"x":-9007199254740993}', "beyond IEEE-754 exact-integer range"), + ("decimal_below_1e_minus_6", '{"x":0.0000001}', "RFC 8785 emits exponent form below 1e-6; outside plain-decimal domain"), + ("lone_high_surrogate", '{"x":"' + LONE_SURROGATE + '"}', "unpaired surrogate is not a Unicode scalar value"), + ("leading_bom", BOM + '{"a":1}', "BOM is not JSON; two lineages silently skipped it"), + ("trailing_garbage", '{"a":1} x', "input must be exactly one JSON document"), + ("two_documents", '{"a":1}{"b":2}', "input must be exactly one JSON document"), + ("trailing_comma", '{"a":1,}', "not valid RFC 8259 JSON"), + ("empty_input", "", "input must be exactly one JSON document"), +] + + +def run_cli(lineage: str, payload: bytes): + cli = ROOT / lineage / "bin" / "baion_canon_hash" + p = subprocess.run([str(cli)], input=payload, capture_output=True) + return p.returncode, p.stdout.decode().strip() + + +def main() -> int: + pin_only = "--pin-only-uniform" in sys.argv + accept_rows, failures = [], [] + for name, text in ACCEPT: + payload = text.encode("utf-8") + results = {L: run_cli(L, payload) for L in LINEAGES} + codes = {L: r[0] for L, r in results.items()} + hashes = {r[1] for r in results.values() if r[0] == 0} + if any(codes.values()) or len(hashes) != 1: + failures.append((name, results)) + if not pin_only: + continue + continue + accept_rows.append({"name": name, "input": text, "sha256": hashes.pop()}) + + (HERE / "accept.jsonl").write_text( + "".join(json.dumps(r, ensure_ascii=True) + "\n" for r in accept_rows)) + (HERE / "reject.jsonl").write_text( + "".join(json.dumps({"name": n, "input": t, "reason": why}, ensure_ascii=True) + "\n" + for n, t, why in REJECT)) + + print(f"pinned {len(accept_rows)}/{len(ACCEPT)} accept cases; {len(REJECT)} reject cases") + for name, results in failures: + detail = " ".join(f"{L}={'REJ' if r[0] else r[1][:8]}" for L, r in results.items()) + print(f"NOT PINNED (divergent/rejecting): {name}: {detail}") + return 1 if (failures and not pin_only) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/reject.jsonl b/conformance/reject.jsonl new file mode 100644 index 0000000..4579510 --- /dev/null +++ b/conformance/reject.jsonl @@ -0,0 +1,19 @@ +{"name": "nul_in_value", "input": "{\"x\":\"a\\u0000b\"}", "reason": "U+0000 not representable losslessly in all lineages"} +{"name": "nul_in_key", "input": "{\"a\\u0000\":1}", "reason": "U+0000 not representable losslessly in all lineages"} +{"name": "dup_key_toplevel", "input": "{\"a\":1,\"a\":2}", "reason": "RFC 8259 duplicate-name behavior undefined; three-way divergence observed"} +{"name": "dup_key_nested", "input": "{\"x\":{\"b\":1,\"b\":2}}", "reason": "duplicate detection required at any depth"} +{"name": "dup_key_escaped", "input": "{\"a\":1,\"\\u0061\":2}", "reason": "duplicates compare on the DECODED name"} +{"name": "dup_key_in_array", "input": "[{\"k\":1,\"k\":2}]", "reason": "duplicate detection inside objects nested in arrays"} +{"name": "exponent_lower", "input": "{\"x\":1e21}", "reason": "exponent tokens outside plain-decimal domain"} +{"name": "exponent_upper", "input": "{\"x\":1E5}", "reason": "exponent tokens outside plain-decimal domain (case-insensitive)"} +{"name": "exponent_tiny", "input": "{\"x\":1e-7}", "reason": "five-way float-format divergence observed"} +{"name": "exponent_overflow", "input": "{\"x\":1e400}", "reason": "overflows double; three accept/four reject divergence observed"} +{"name": "int_beyond_2_53", "input": "{\"x\":9007199254740993}", "reason": "beyond IEEE-754 exact-integer range"} +{"name": "neg_int_beyond_2_53", "input": "{\"x\":-9007199254740993}", "reason": "beyond IEEE-754 exact-integer range"} +{"name": "decimal_below_1e_minus_6", "input": "{\"x\":0.0000001}", "reason": "RFC 8785 emits exponent form below 1e-6; outside plain-decimal domain"} +{"name": "lone_high_surrogate", "input": "{\"x\":\"\\ud800\"}", "reason": "unpaired surrogate is not a Unicode scalar value"} +{"name": "leading_bom", "input": "\ufeff{\"a\":1}", "reason": "BOM is not JSON; two lineages silently skipped it"} +{"name": "trailing_garbage", "input": "{\"a\":1} x", "reason": "input must be exactly one JSON document"} +{"name": "two_documents", "input": "{\"a\":1}{\"b\":2}", "reason": "input must be exactly one JSON document"} +{"name": "trailing_comma", "input": "{\"a\":1,}", "reason": "not valid RFC 8259 JSON"} +{"name": "empty_input", "input": "", "reason": "input must be exactly one JSON document"} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a2ed3a1..43ff60d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,7 +1,7 @@ # BAION canonical JSON + SHA-256 for C++ — public standalone library. cmake_minimum_required(VERSION 3.20) project(baion_std_cpp_public - VERSION 0.1.0 + VERSION 0.2.0 LANGUAGES CXX DESCRIPTION "BAION canonical JSON + SHA-256 (C++)" ) diff --git a/cpp/include/baion/canonical_json.hpp b/cpp/include/baion/canonical_json.hpp index c68d026..04cfd06 100644 --- a/cpp/include/baion/canonical_json.hpp +++ b/cpp/include/baion/canonical_json.hpp @@ -45,6 +45,31 @@ bool contains_nul(const nlohmann::json& j); // the decoded-name semantics of the contract. bool has_duplicate_keys(const std::string& raw_input); +// ── Number-domain rejection scan ────────────────────────────── +// CROSS-LINEAGE CONTRACT: any input containing a number outside +// the supported canonical domain must be rejected before +// canonicalization. Returns true if any number (at any depth) is: +// - written with an exponent (raw token contains 'e' or 'E'), +// e.g. 1e2 is rejected even though 100 is accepted — the check +// is lexical, on the source spelling, not the value; +// - an integer beyond +/-9007199254740992 (2^53, the largest +// magnitude every lineage's double can hold exactly); +// - a fraction with magnitude in (0, 1e-6) or >= 1e21, where +// lineage formatters diverge (scientific-notation thresholds). +// Note: this operates on the RAW input text via nlohmann's SAX +// interface, because number_float() receives the unmodified source +// token alongside the parsed value — the DOM erases the spelling. +bool has_unsupported_number(const std::string& raw_input); + +// ── UTF-8 BOM rejection scan ────────────────────────────────── +// CROSS-LINEAGE CONTRACT: a leading UTF-8 byte-order mark +// (EF BB BF) is rejected, not skipped — RFC 8259 §8.1 forbids +// adding a BOM, and silently stripping it would let two byte- +// distinct inputs hash identically. Checks the first three raw +// bytes only; a BOM anywhere else is ordinary string content for +// the parser to judge. +bool has_utf8_bom(const std::string& raw_input); + // ── Checked canonicalization (library error path) ───────────── // Rejection-aware entry point: scans for U+0000 first, then // canonicalizes into `out`. Returns false (leaving `out` empty) diff --git a/cpp/src/canonical_json.cpp b/cpp/src/canonical_json.cpp index d117482..942563e 100644 --- a/cpp/src/canonical_json.cpp +++ b/cpp/src/canonical_json.cpp @@ -6,8 +6,11 @@ #include "baion/canonical_json.hpp" #include +#include #include +#include #include +#include #include #include @@ -82,10 +85,8 @@ static void serialize_number(const nlohmann::json& j, std::string& out) } else { - // Floating point — use nlohmann's dump which produces - // minimal representation. We need to ensure no trailing .0 - // for whole numbers, but nlohmann handles this. - // However, for cross-lineage consistency we use a fixed approach: + // Floating point — ES-262 ToString restricted to plain decimal + // (never nlohmann's dump(), which switches to exponent notation). double val = j.get(); if (std::isnan(val) || std::isinf(val)) { @@ -105,8 +106,78 @@ static void serialize_number(const nlohmann::json& j, std::string& out) } else { - // Use nlohmann's serializer for consistent representation - out += j.dump(); + // CROSS-LINEAGE CONTRACT: non-integer floats serialize as the + // SHORTEST decimal string that roundtrips to the same double + // (RFC 8785 §3.2.2.3 / ECMA-262 §7.1.12.1), reassembled WITHOUT + // exponent notation. The number-domain gate guarantees the value + // is 0 or |v| in [1e-6, 1e21), which is exactly the range where + // ECMA-262 ToString never takes its exponent branch — so plain + // positional layout is the canonical spelling. nlohmann's dump() + // here previously printed 1e-05 for 0.00001 and 1e+20 near the + // domain top, diverging from every RFC 8785 lineage. + + // Shortest digits: std::to_chars with chars_format::scientific + // and default precision emits the minimal round-trip digit + // string as d[.ddd]e±dd — locale-independent, unlike the %e + // loop the C lineage uses for the same contract. + char sci[64]; + const auto rc = std::to_chars(sci, sci + sizeof(sci) - 1, val, + std::chars_format::scientific); + *rc.ptr = '\0'; + + // Pull apart [-]d[.ddd]e±dd into digit string D and + // n = exp10 + 1 (count of digits before the decimal point in + // positional form), per ECMA-262 §7.1.12.1 notation. + const char* s = sci; + const bool neg = (*s == '-'); + if (neg) + ++s; + std::string digits; + digits.push_back(*s++); + if (*s == '.') + { + ++s; + while (*s != 'e' && *s != 'E') + digits.push_back(*s++); + } + ++s; // skip 'e'; strtol consumes the +/- sign of the exponent + const int n = static_cast(std::strtol(s, nullptr, 10)) + 1; + const int dl = static_cast(digits.size()); + + if (neg) + out.push_back('-'); + if (dl <= n && n <= 21) + { + // Integer-valued but too large for the int64 branch above + // (|v| >= 1e15): all digits then zero-padding, no dot. + out += digits; + out.append(static_cast(n - dl), '0'); + } + else if (0 < n && n <= dl) + { + out.append(digits, 0, static_cast(n)); + out.push_back('.'); + out.append(digits, static_cast(n), + std::string::npos); + } + else if (-5 <= n && n <= 0) + { + out += "0."; + out.append(static_cast(-n), '0'); + out += digits; + } + else + { + // Only reachable when a caller bypassed the number-domain + // gate (programmatic tree with |v| >= 1e21 or |v| < 1e-6): + // emit a best-effort spelling rather than mis-slice the + // digit string. NON-CANONICAL. + if (neg) + out.pop_back(); + char buf[64]; + snprintf(buf, sizeof(buf), "%.17g", val); + out += buf; + } } } } @@ -223,6 +294,143 @@ bool has_duplicate_keys(const std::string& raw_input) return scanner.found_duplicate; } +// ── Number-domain rejection scan ─────────────────────────────── +// CROSS-LINEAGE CONTRACT: all lineages reject numbers outside the +// canonical domain (exponent spellings, integers beyond ±2^53, +// fractions in (0, 1e-6) or >= 1e21). The exponent check must be +// lexical — 100 and 1e2 are the same value but different tokens — +// so this scan uses the SAX interface: number_float() receives the +// raw source token, which the DOM parse discards. +namespace +{ +class number_domain_scanner final + : public nlohmann::json_sax +{ +public: + bool found_unsupported = false; + + // Largest integer magnitude representable exactly in a double + // across every lineage: 2^53. int64/uint64 comparison is exact — + // no float conversion happens on this path. + static constexpr std::int64_t kMaxSafe = 9007199254740992LL; + + bool number_integer(number_integer_t val) override + { + if (val > kMaxSafe || val < -kMaxSafe) + { + found_unsupported = true; + return false; // abort the parse — one bad number is enough + } + return true; + } + + bool number_unsigned(number_unsigned_t val) override + { + if (val > static_cast(kMaxSafe)) + { + found_unsupported = true; + return false; + } + return true; + } + + bool number_float(number_float_t val, const string_t& raw) override + { + // Lexical exponent check: nlohmann routes any token with an + // exponent or decimal point to number_float and hands us the + // unmodified source spelling, so 1e2 is caught here while the + // integer token 100 never reaches this callback. + if (raw.find('e') != std::string::npos || + raw.find('E') != std::string::npos) + { + found_unsupported = true; + return false; + } + // A dotless raw token here is an INTEGER that overflowed the + // 64-bit integer callbacks (nlohmann falls back to + // number_float on u64/i64 overflow) — the fraction window + // below would wrongly admit e.g. 100000000000000000000 + // (1e20 < 1e21). Judge it as the integer callbacks do: + // digit-string comparison against 2^53, exact at any length. + if (raw.find('.') == std::string::npos) + { + if (integer_token_exceeds_max_safe(raw)) + { + found_unsupported = true; + return false; + } + return true; + } + // Magnitude window where all lineage formatters agree on a + // plain (non-scientific) decimal rendering. + if ((val != 0.0 && std::fabs(val) < 1e-6) || + std::fabs(val) >= 1e21) + { + found_unsupported = true; + return false; + } + return true; + } + + // Compares the raw digit string against 2^53 without any numeric + // conversion — tokens on this path already overflowed uint64, so + // only string arithmetic is exact. + static bool integer_token_exceeds_max_safe(const string_t& raw) + { + static const std::string kMaxSafeDigits = "9007199254740992"; + std::size_t i = 0; + if (i < raw.size() && raw[i] == '-') + ++i; + while (i + 1 < raw.size() && raw[i] == '0') + ++i; + const std::string digits = raw.substr(i); + if (digits.size() != kMaxSafeDigits.size()) + return digits.size() > kMaxSafeDigits.size(); + return digits > kMaxSafeDigits; + } + + // Remaining events carry no number information — accept and continue. + bool null() override { return true; } + bool boolean(bool) override { return true; } + bool string(string_t&) override { return true; } + bool binary(binary_t&) override { return true; } + bool key(string_t&) override { return true; } + bool start_object(std::size_t) override { return true; } + bool end_object() override { return true; } + bool start_array(std::size_t) override { return true; } + bool end_array() override { return true; } + + bool parse_error(std::size_t, const std::string&, + const nlohmann::detail::exception&) override + { + // Malformed input is the DOM parse's error to report, not + // ours — stop scanning without flagging a number. + return false; + } +}; +} // namespace + +bool has_unsupported_number(const std::string& raw_input) +{ + number_domain_scanner scanner; + // Return value ignored deliberately: sax_parse also returns false + // on plain syntax errors, and only found_unsupported answers the + // question this scan asks. + nlohmann::json::sax_parse(raw_input, &scanner); + return scanner.found_unsupported; +} + +// ── UTF-8 BOM rejection scan ─────────────────────────────────── +// CROSS-LINEAGE CONTRACT: nlohmann silently skips a leading BOM, +// so a BOM-prefixed document would otherwise hash identically to +// its BOM-free twin — the check must run on the raw bytes before +// any parse touches them. +bool has_utf8_bom(const std::string& raw_input) +{ + return raw_input.size() >= 3 && raw_input[0] == '\xEF' && + raw_input[1] == '\xBB' && raw_input[2] == '\xBF'; +} + // ── Checked canonicalization (library error path) ───────────── // Bool-return sentinel because the library is built with // -fno-exceptions; callers branch on the return value. diff --git a/cpp/tests/test_canonical_json.cpp b/cpp/tests/test_canonical_json.cpp index 4692a4e..29911b5 100644 --- a/cpp/tests/test_canonical_json.cpp +++ b/cpp/tests/test_canonical_json.cpp @@ -43,7 +43,7 @@ TEST(CanonicalJSON, MinimalStringEscaping) // Forward slash is NOT escaped (minimal escaping) EXPECT_NE(result.find("just ascii / and more"), std::string::npos); - // Control char 0x01 escaped as  + // Control char 0x01 escaped as the six-character text \u0001 EXPECT_NE(result.find("\\u0001"), std::string::npos); // Quote escaped EXPECT_NE(result.find("\\\"hello\\\""), std::string::npos); @@ -187,6 +187,88 @@ TEST(CanonicalJSON, AllowsDistinctKeys) EXPECT_FALSE(has_duplicate_keys("{\"a\":{\"a\":1},\"b\":{\"a\":2}}")); } +// ── Number-domain rejection: exponent spelling is lexical ───── +// 100 and 1e2 denote the same value; the contract rejects the +// exponent SPELLING, so the check must see the raw token — the +// SAX number_float callback delivers it. +TEST(CanonicalJSON, RejectsExponentNotation) +{ + EXPECT_TRUE(has_unsupported_number("{\"x\":1e2}")); + EXPECT_TRUE(has_unsupported_number("{\"x\":1E5}")); + EXPECT_TRUE(has_unsupported_number("{\"x\":1e-7}")); + EXPECT_TRUE(has_unsupported_number("{\"x\":2.5E+3}")); + // Same value, integer spelling — accepted. + EXPECT_FALSE(has_unsupported_number("{\"x\":100}")); +} + +// ── Number-domain rejection: integers beyond +/-2^53 ────────── +// 9007199254740992 (2^53) is the last integer every lineage's +// double holds exactly; one past it in either direction is out. +// The comparison is int64/uint64-exact — no float rounding. +TEST(CanonicalJSON, RejectsIntegersBeyondSafeRange) +{ + EXPECT_TRUE(has_unsupported_number("{\"x\":9007199254740993}")); + EXPECT_TRUE(has_unsupported_number("{\"x\":-9007199254740993}")); + EXPECT_FALSE(has_unsupported_number("{\"x\":9007199254740992}")); + EXPECT_FALSE(has_unsupported_number("{\"x\":-9007199254740992}")); +} + +// ── Number-domain rejection: integers past uint64 ────────────── +// nlohmann routes integers that overflow 64 bits to number_float, +// bypassing the integer callbacks — the scanner must recognize a +// dotless raw token as an integer and judge it by digit string, +// not by the fraction magnitude window (1e20 < 1e21 would pass). +TEST(CanonicalJSON, RejectsIntegersBeyondUint64) +{ + EXPECT_TRUE(has_unsupported_number( + "{\"x\":100000000000000000000}")); + EXPECT_TRUE(has_unsupported_number( + "{\"x\":-100000000000000000000}")); + // 2^64 + 1: just past uint64, well past 2^53. + EXPECT_TRUE(has_unsupported_number( + "{\"x\":18446744073709551617}")); + // A genuine fraction token (has '.') near the top of the window + // stays on the existing magnitude check and remains accepted. + EXPECT_FALSE(has_unsupported_number( + "{\"x\":100000000000000000000.5}")); +} + +// ── Number-domain rejection: out-of-range fractions ─────────── +// Below 1e-6 (nonzero) or at/above 1e21 lineage formatters flip +// to scientific notation and diverge; the domain excludes them. +TEST(CanonicalJSON, RejectsOutOfRangeFractions) +{ + EXPECT_TRUE(has_unsupported_number("{\"x\":0.0000001}")); + // Boundary stays in: exactly 1e-6 written plainly is supported. + EXPECT_FALSE(has_unsupported_number("{\"x\":0.000001}")); + EXPECT_FALSE(has_unsupported_number("{\"x\":0.1}")); + EXPECT_FALSE(has_unsupported_number("{\"x\":0.0}")); + EXPECT_FALSE(has_unsupported_number("{\"x\":-0.0}")); +} + +// ── Number-domain rejection: bad number at depth ────────────── +TEST(CanonicalJSON, RejectsUnsupportedNumberAtDepth) +{ + EXPECT_TRUE(has_unsupported_number("{\"a\":[1,{\"b\":[2,1e2]}]}")); + EXPECT_FALSE(has_unsupported_number( + "{\"nested\":{\"y\":[true,false,null],\"x\":0.5},\"arr\":[]}")); +} + +// ── UTF-8 BOM rejection: leading BOM caught, elsewhere ignored ─ +// RFC 8259 §8.1 forbids adding a BOM; nlohmann would silently skip +// it, so the raw-byte check must fire before any parse. +TEST(CanonicalJSON, RejectsLeadingUtf8Bom) +{ + EXPECT_TRUE(has_utf8_bom("\xEF\xBB\xBF{\"a\":1}")); + EXPECT_TRUE(has_utf8_bom("\xEF\xBB\xBF")); + EXPECT_FALSE(has_utf8_bom("{\"a\":1}")); + // Too short to hold a BOM, and prefixes of one are not a BOM. + EXPECT_FALSE(has_utf8_bom("")); + EXPECT_FALSE(has_utf8_bom("\xEF\xBB")); + // BOM bytes past position 0 are ordinary content for the parser. + EXPECT_FALSE(has_utf8_bom(" \xEF\xBB\xBF")); +} + // ── Checked path matches unchecked path on clean input ──────── TEST(CanonicalJSON, CheckedMatchesUncheckedOnCleanInput) { @@ -222,3 +304,44 @@ TEST(CanonicalJSON, IntegerValuedFloats) EXPECT_EQ(result, expected); } + +// ── Floats: shortest round-trip digits, plain positional layout ── +// Each case: shortest decimal string that roundtrips to the same double, +// reassembled WITHOUT exponent notation, per RFC 8785 / ECMA-262 +// §7.1.12.1. The old nlohmann dump() path printed 0.00001 as "1e-05" +// and 1e20-scale values as "1e+20", diverging from every other lineage. +TEST(CanonicalJSON, FloatShortestRoundtripPlainDecimal) +{ + static const struct + { + double v; + const char* expect; + } cases[] = { + {0.1, "0.1"}, + {123.456, "123.456"}, + {0.5, "0.5"}, + {0.001, "0.001"}, + {0.0001, "0.0001"}, + {0.000123, "0.000123"}, + {0.00001, "0.00001"}, + {0.000001, "0.000001"}, // n = -5 boundary: "0." + 5 zeros + D + {1e16, "10000000000000000"}, // dl < n: zero-padded, no dot + {9007199254740992.0, + "9007199254740992"}, // 2^53: above the int64 branch cutoff + // 1e20 + 0.5 rounds to the 21-digit integer double 1e20: exercises + // the n = 21 top of the zero-padding branch (domain ceiling). + {100000000000000000000.5, "100000000000000000000"}, + {-2.5, "-2.5"}, + {-0.375, "-0.375"}, + {1.0, "1"}, // integer-valued keeps no-trailing-".0" behavior + {-0.0, "0"}, // zero (incl. negative zero) always emits exactly "0" + }; + + for (const auto& c : cases) + { + nlohmann::json n = c.v; + ASSERT_TRUE(n.is_number_float()) + << "case must be float-tagged: " << c.expect; + EXPECT_EQ(canonicalize_json(n), c.expect); + } +} diff --git a/cpp/tools/baion_canon_hash.cpp b/cpp/tools/baion_canon_hash.cpp index 053e2a9..7ad6d98 100644 --- a/cpp/tools/baion_canon_hash.cpp +++ b/cpp/tools/baion_canon_hash.cpp @@ -3,8 +3,11 @@ // Reads UTF-8 JSON on stdin, canonicalizes it with the BAION // canonical JSON rules, prints the lowercase-hex SHA-256 of the // canonical bytes followed by a newline. Exit 0 on success, -// nonzero on parse error, on U+0000 anywhere in a string, or on -// a duplicate object key (decoded names) at any depth. +// nonzero on parse error, on a leading UTF-8 BOM, on U+0000 +// anywhere in a string, on a duplicate object key (decoded names) +// at any depth, or on a number outside the canonical domain +// (exponent spelling, integer beyond +/-2^53, out-of-range +// fraction). #include "baion/canonical_json.hpp" #include "baion/hash.hpp" @@ -20,6 +23,17 @@ int main() std::string input((std::istreambuf_iterator(std::cin)), std::istreambuf_iterator()); + // BOM scan on the RAW bytes before any parse — nlohmann would + // silently skip a leading UTF-8 BOM, letting a BOM-prefixed + // document hash identically to its BOM-free twin. + if (baion::std_lib::has_utf8_bom(input)) + { + std::fprintf(stderr, + "baion_canon_hash: input starts with a UTF-8 BOM — " + "unsupported, rejected\n"); + return 1; + } + // Non-throwing parse (library is built with -fno-exceptions). nlohmann::json j = nlohmann::json::parse(input, nullptr, false); if (j.is_discarded()) @@ -39,6 +53,18 @@ int main() return 1; } + // Number-domain scan on the RAW input — the exponent check is + // lexical (1e2 rejected, 100 accepted), so it needs the source + // tokens the DOM parse discards (cross-lineage contract). + if (baion::std_lib::has_unsupported_number(input)) + { + std::fprintf(stderr, + "baion_canon_hash: input contains an unsupported " + "number (exponent notation, integer beyond +/-2^53, " + "or out-of-range fraction) — rejected\n"); + return 1; + } + // Checked canonicalization — rejects any object key or string // value containing U+0000 (cross-lineage contract). std::string canonical; diff --git a/d/source/baionstd/canonical_json.d b/d/source/baionstd/canonical_json.d index eb201b6..0fcd6a5 100644 --- a/d/source/baionstd/canonical_json.d +++ b/d/source/baionstd/canonical_json.d @@ -15,63 +15,97 @@ import std.array : Appender, appender; import std.conv : to; import std.format : format; import std.math : isNaN, isInfinity; +import std.typecons : Nullable; import baionstd.types : StdError, errorMessage; -// CROSS-LINEAGE CONTRACT: this formatting must produce byte-identical output -// with Rust's Ryu (serde_json) and C++'s nlohmann::json shortest-round-trip. -// All lineages must agree on the decimal representation so that -// canonical JSON and therefore SHA-256 digests match. +// CROSS-LINEAGE CONTRACT: non-integer floats serialize as the SHORTEST +// decimal string that roundtrips to the same double (RFC 8785 §3.2.2.3 / +// ECMA-262 §7.1.12.1), reassembled WITHOUT exponent notation. The +// number-domain gate guarantees the value is 0 or |v| in [1e-6, 1e21), +// which is exactly the range where ECMA-262 ToString never takes its +// exponent branch — so plain positional layout is the canonical spelling. +// The previous %g-based formatter emitted exponent forms at |v| <= 1e-5 +// (0.00001 → "1e-5") and near the 1e21 top, diverging from the C/Go/OCaml +// reference lineages and breaking SHA-256 digest parity. // -// WHY: D's %g diverges from Ryu in two ways that must be post-processed: -// 1. Zero-pads exponents: 5e-08 vs Ryu's 5e-8 -// 2. Includes + sign: e+15 vs Ryu omits + -// The %g shortest-search itself matches Ryu for all values in the -// pipeline's float range. -/// Iterates %g precision from 1..17 until parse(format(val)) == val, -/// then normalizes exponent formatting to match Ryu. +// Locale note: std.format implements float formatting in Phobos (always +// '.' decimal point) and to!double likewise parses locale-independently, +// so unlike the C lineage's snprintf/strtod path this code does not +// depend on the process locale. +/// Shortest round-trip digits via %e precision search (p = 1..17), +/// then ECMA-262 §7.1.12.1 plain-decimal reassembly. private string formatShortest(double val) { + // Shortest digits: smallest precision whose %e output parses back + // to the identical double. + string sci; foreach (prec; 1 .. 18) { - string s = format("%.*g", prec, val); - if (to!double(s) == val) - return normalizeExponent(s); + sci = format("%.*e", prec - 1, val); + if (to!double(sci) == val) + break; } - // Fallback: full 17-digit precision (unreachable for finite IEEE-754 doubles) - return normalizeExponent(format!"%.17g"(val)); -} -/// Post-process %g output to match Ryu's exponent format. -/// WHY: D's libc %g zero-pads exponents (5e-08 → 5e-8) and includes -/// + sign (e+15 → e15). Ryu never does either. -private string normalizeExponent(string s) -{ - import std.string : indexOf; - - auto eIdx = s.indexOf('e'); - if (eIdx < 0) - eIdx = s.indexOf('E'); - if (eIdx < 0) - return s; // No exponent, no fixup needed - - string mantissa = s[0 .. eIdx]; - string expPart = s[eIdx + 1 .. $]; - - // Parse exponent: strip + sign and leading zeros - bool negExp = false; - size_t ei = 0; - if (ei < expPart.length && expPart[ei] == '+') - ei++; - else if (ei < expPart.length && expPart[ei] == '-') + // Pull apart [-]d[.ddd]e±XX into digit string D (mantissa digits, + // no dot) and n = exp10 + 1 (count of digits before the decimal + // point in positional form). The exponent is parsed numerically: + // %e implementations emit 2+ exponent digits and a mandatory sign, + // so no fixed width can be assumed. + size_t i = 0; + bool neg = false; + if (sci[i] == '-') + { + neg = true; + i++; + } + auto digitsBuf = appender!string; + digitsBuf.put(sci[i]); + i++; + if (i < sci.length && sci[i] == '.') { - negExp = true; - ei++; + i++; + while (sci[i] != 'e' && sci[i] != 'E') + { + digitsBuf.put(sci[i]); + i++; + } } - while (ei < expPart.length - 1 && expPart[ei] == '0') - ei++; + i++; // skip 'e'; to!int consumes the +/- sign and leading zeros + immutable int n = to!int(sci[i .. $]) + 1; + + string digits = digitsBuf[]; + immutable int dl = cast(int) digits.length; - return mantissa ~ "e" ~ (negExp ? "-" : "") ~ expPart[ei .. $]; + string outp; + if (dl <= n && n <= 21) + { + // Integer-valued: all digits then zero-padding, no dot. Covers + // integer-valued doubles too large for exact-precision layout + // (e.g. 1.000000000000000005e20 rounds to a 21-digit integer). + outp = digits; + foreach (_; 0 .. n - dl) + outp ~= '0'; + } + else if (0 < n && n <= dl) + { + outp = digits[0 .. n] ~ "." ~ digits[n .. $]; + } + else if (-5 <= n && n <= 0) + { + outp = "0."; + foreach (_; 0 .. -n) + outp ~= '0'; + outp ~= digits; + } + else + { + // Only reachable when a caller bypassed the number-domain gate + // (programmatic JSONValue with |v| >= 1e21 or nonzero |v| < 1e-6): + // emit a best-effort spelling rather than fail. NON-CANONICAL. + return format!"%.17g"(val); + } + return neg ? "-" ~ outp : outp; } /// Convert any JSONValue to canonical JSON string. @@ -113,13 +147,21 @@ void canonicalizeValue(ref Appender!string buf, const JSONValue v) { buf.put("null"); } + else if (val == 0) + { + // CROSS-LINEAGE CONTRACT: any zero — including IEEE-754 negative + // zero — serializes as exactly "0" (RFC 8785 §3.2.2.3 via ECMA-262 + // ToString: "If x is +0 or -0, return \"0\""). D's %g preserves + // the sign bit and would emit "-0", diverging from the other + // lineages, so zero is short-circuited before formatShortest. + buf.put("0"); + } else { // CROSS-LINEAGE CONTRACT: integer-valued floats serialize without - // trailing decimal (RFC 8785 §3.2.2.3 / ECMA-262 §7.1.12.1). The - // %g-based formatShortest below performs the shortest-round-trip - // search starting at precision 1, so 1.0 → "1", -3.0 → "-3", and - // 1.5 → "1.5". + // trailing decimal (RFC 8785 §3.2.2.3 / ECMA-262 §7.1.12.1). + // formatShortest's precision-1 starting point plus the dl <= n + // reassembly branch guarantee 1.0 → "1", -3.0 → "-3", 1.5 → "1.5". buf.put(formatShortest(val)); } break; @@ -251,6 +293,232 @@ bool hasDuplicateKeys(const(char)[] raw) return false; } +// ── Number-domain enforcement scan ─────────────────────────── +// +// CROSS-LINEAGE CONTRACT: number tokens are restricted to a lexical +// domain all 7 lineages accept identically — the check is on the RAW +// token text, so `100` and `1e2` are distinguished even though they +// parse to the same value. Rejected (CLI exits 1 mentioning an +// unsupported number): +// 1. any token containing exponent notation (`e` / `E`) +// 2. integer tokens (no `.`) with magnitude beyond ±9007199254740992 +// (2^53, the IEEE-754 exact-integer bound) — compared as digit +// strings, never via double, so 9007199254740993 cannot round +// down to a false accept +// 3. fraction tokens whose value lands where ES ToString would need +// exponent form: nonzero |v| < 1e-6 or |v| >= 1e21 +// +// WHY a raw-input scan and not a post-parse walk: parseJSON has +// already collapsed `1e2` to 100 and rounded 9007199254740993 by the +// time a JSONValue exists — the lexical evidence is gone. +// +// Precondition: as with hasDuplicateKeys, the input has already been +// accepted by parseJSON, so tokens are well-formed JSON numbers. + +/// Scan raw JSON text for number tokens outside the supported domain. +/// Returns true if any unsupported number token exists. +bool hasUnsupportedNumber(const(char)[] raw) +{ + size_t i = 0; + while (i < raw.length) + { + char c = raw[i]; + if (c == '"') + { + // Digits/'e' inside string literals are not number tokens: + // consume the whole literal (decoded text is discarded). + decodeJSONString(raw, i); + continue; + } + if (c == '-' || (c >= '0' && c <= '9')) + { + size_t start = i; + // parseJSON already validated the grammar, so a greedy sweep + // over number-alphabet chars captures exactly one token. + while (i < raw.length && (raw[i] == '-' || raw[i] == '+' + || raw[i] == '.' || raw[i] == 'e' || raw[i] == 'E' + || (raw[i] >= '0' && raw[i] <= '9'))) + i++; + if (numberTokenUnsupported(raw[start .. i])) + return true; + continue; + } + i++; + } + return false; +} + +/// Decide whether one raw number token is outside the supported domain. +private bool numberTokenUnsupported(const(char)[] tok) +{ + import std.conv : to; + import std.math : fabs; + + bool hasDot = false; + foreach (c; tok) + { + // Exponent notation is rejected outright — even value-preserving + // forms like 1e2 — because canonical output would erase the + // distinction and lineages differ in how they re-expand it. + if (c == 'e' || c == 'E') + return true; + if (c == '.') + hasDot = true; + } + + if (!hasDot) + { + // Integer token: compare DIGIT STRINGS against 2^53. Converting + // to double first would round 9007199254740993 down to the + // boundary and wave it through. + static immutable string maxSafe = "9007199254740992"; + const(char)[] digits = tok; + if (digits.length && digits[0] == '-') + digits = digits[1 .. $]; + while (digits.length > 1 && digits[0] == '0') + digits = digits[1 .. $]; + if (digits.length > maxSafe.length) + return true; + if (digits.length == maxSafe.length && digits > maxSafe) + return true; + return false; + } + + // Fraction token: magnitude gate on the parsed value. Outside + // [1e-6, 1e21) ES ToString switches to exponent form, which the + // supported domain excludes. Zero (any sign) always passes. + double v = to!double(tok); + return (v != 0 && fabs(v) < 1e-6) || fabs(v) >= 1e21; +} + +// ── Single-document enforcement scan ───────────────────────── +// +// CROSS-LINEAGE CONTRACT: stdin must contain EXACTLY ONE complete JSON +// document, with only leading/trailing whitespace around it. All 7 +// lineages reject empty input, trailing garbage (`{"a":1} x`), +// concatenated documents (`{"a":1}{"b":2}`) and trailing commas +// (`{"a":1,}`) — the CLI exits 1 naming the problem. +// +// WHY a raw-input scan: std.json's parseJSON stops at the end of the +// first complete value and silently IGNORES everything after it, and +// (measured, dmd 2.112.0) also swallows a trailing comma before '}' +// or ']' — so by the time a JSONValue exists none of these defects +// are visible. The consuming-range parseJSON overload does not help +// either: for a char slice it leaves the range untouched rather than +// advancing past the parsed prefix. This scanner walks the raw text +// to the end of the first document and checks the remainder is +// whitespace-only, flagging trailing commas along the way. +// +// Precondition: as with the other raw scanners, parseJSON has already +// accepted the input, so the FIRST document is well-formed; the scan +// never reads past it except to whitespace-check the remainder. + +/// Scan raw JSON text for the exactly-one-document contract. +/// Returns the violation, or a null Nullable when the input is one +/// complete document surrounded only by whitespace. +Nullable!StdError scanSingleDocument(const(char)[] raw) +{ + Nullable!StdError err; + + size_t i = 0; + while (i < raw.length && isJSONWhitespace(raw[i])) + i++; + if (i >= raw.length) + { + err = StdError.emptyInput; + return err; + } + + char c = raw[i]; + if (c == '{' || c == '[') + { + // Structural walk to the matching top-level closer. Strings are + // consumed whole so structural bytes inside literals never count. + // afterComma tracks whether the last non-whitespace structural + // token was ',' — true at a closer means a trailing comma, which + // parseJSON accepts but the contract rejects. + size_t depth = 0; + bool afterComma = false; + while (i < raw.length) + { + char t = raw[i]; + if (t == '"') + { + decodeJSONString(raw, i); + afterComma = false; + continue; + } + if (t == '{' || t == '[') + { + depth++; + afterComma = false; + i++; + continue; + } + if (t == '}' || t == ']') + { + if (afterComma) + { + err = StdError.trailingComma; + return err; + } + depth--; + i++; + if (depth == 0) + break; + continue; + } + if (t == ',') + afterComma = true; + else if (!isJSONWhitespace(t)) + afterComma = false; // ':' and scalar bytes clear the flag + i++; + } + } + else if (c == '"') + { + decodeJSONString(raw, i); + } + else if (c == 't') + { + i += 4; // "true" — exact literal, parseJSON already validated it + } + else if (c == 'f') + { + i += 5; // "false" + } + else if (c == 'n') + { + i += 4; // "null" + } + else + { + // Number token: greedy sweep over the number alphabet, matching + // hasUnsupportedNumber. Nonempty for parseJSON-accepted input; + // exact token extent matters so `123 456` and `123x` leave their + // second chunk behind as trailing data. + size_t start = i; + while (i < raw.length && (raw[i] == '-' || raw[i] == '+' + || raw[i] == '.' || raw[i] == 'e' || raw[i] == 'E' + || (raw[i] >= '0' && raw[i] <= '9'))) + i++; + if (i == start) + i++; // unreachable for parseJSON-accepted input; keeps the scan advancing + } + + while (i < raw.length && isJSONWhitespace(raw[i])) + i++; + if (i < raw.length) + err = StdError.trailingData; + return err; +} + +/// RFC 8259 §2 insignificant whitespace: space, tab, LF, CR — nothing else. +private bool isJSONWhitespace(char c) @safe pure nothrow +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + /// Decode one JSON string literal starting at the opening quote. /// Advances `i` past the closing quote; returns the decoded text. /// WHY decode at all: key comparison must match the parser's view, diff --git a/d/source/baionstd/types.d b/d/source/baionstd/types.d index f7c40c1..5c17bd3 100644 --- a/d/source/baionstd/types.d +++ b/d/source/baionstd/types.d @@ -8,6 +8,10 @@ enum StdError malformedMessage, /// Invalid JSON nulInString, /// U+0000 in a string value or object key duplicateKey, /// Duplicate member name (decoded) within one object + unsupportedNumber, /// Number token outside the supported lexical domain + emptyInput, /// Empty or whitespace-only input (no JSON document at all) + trailingData, /// Content after the first complete JSON document + trailingComma, /// Comma immediately before '}' or ']' } /// Human-readable error messages. @@ -21,5 +25,13 @@ string errorMessage(StdError e) @safe pure nothrow return "U+0000 in string value or object key is rejected"; case StdError.duplicateKey: return "duplicate object key (decoded name) is rejected"; + case StdError.unsupportedNumber: + return "unsupported number (exponent notation or out-of-range magnitude) is rejected"; + case StdError.emptyInput: + return "empty input is rejected (exactly one JSON document is required)"; + case StdError.trailingData: + return "trailing data after the JSON document is rejected (exactly one JSON document is required)"; + case StdError.trailingComma: + return "trailing comma before '}' or ']' is rejected"; } } diff --git a/d/tests/conformance_test.d b/d/tests/conformance_test.d index f40782b..a386f25 100644 --- a/d/tests/conformance_test.d +++ b/d/tests/conformance_test.d @@ -109,9 +109,52 @@ unittest JSONValue nan = JSONValue(double.nan); assert(canonicalizeJSON(nan) == "null", "edge case FAILED: NaN → null"); - // Exponent normalization (Ryu format: no +, no zero-padding) - JSONValue tiny = JSONValue(5e-8); - assert(canonicalizeJSON(tiny) == "5e-8", "edge case FAILED: exponent normalization"); +} + +// ── ES ToString plain-decimal float spelling (RFC 8785 §3.2.2.3) ── +// The number-domain gate restricts fractions to |v| in [1e-6, 1e21), +// exactly where ECMA-262 ToString never uses exponent form — canonical +// output is therefore ALWAYS plain positional decimal. The previous +// %g-based formatter emitted "1e-5" at |v| <= 1e-5, diverging from the +// C/Go/OCaml reference lineages. +unittest +{ + // Small-fraction band (0 < n <= 0 reassembly): leading "0." + zeros + assert(canonicalizeJSON(JSONValue(0.001)) == "0.001", + "plain-decimal FAILED: 0.001"); + assert(canonicalizeJSON(JSONValue(0.0001)) == "0.0001", + "plain-decimal FAILED: 0.0001"); + assert(canonicalizeJSON(JSONValue(0.000123)) == "0.000123", + "plain-decimal FAILED: 0.000123"); + assert(canonicalizeJSON(JSONValue(0.00001)) == "0.00001", + "plain-decimal FAILED: 0.00001 (old formatter emitted 1e-5)"); + assert(canonicalizeJSON(JSONValue(0.000001)) == "0.000001", + "plain-decimal FAILED: 0.000001 domain floor (old formatter emitted 1e-6)"); + + // Mid-range split (0 < n <= dl reassembly) + assert(canonicalizeJSON(JSONValue(123.456)) == "123.456", + "plain-decimal FAILED: 123.456"); + assert(canonicalizeJSON(JSONValue(0.25)) == "0.25", + "plain-decimal FAILED: 0.25"); + assert(canonicalizeJSON(JSONValue(-0.375)) == "-0.375", + "plain-decimal FAILED: -0.375"); + + // Near the 1e21 top (dl <= n <= 21 reassembly): 100000000000000000000.5 + // rounds to an integer-valued double that needs zero-padding to its + // 21-digit integer spelling, never exponent form. + assert(canonicalizeJSON(JSONValue(100000000000000000000.5)) + == "100000000000000000000", + "plain-decimal FAILED: 21-digit integer-valued double near 1e21 top"); + // Integer-valued double above 2^53 (large but exactly representable) + assert(canonicalizeJSON(JSONValue(1e15)) == "1000000000000000", + "plain-decimal FAILED: 1e15 integer-valued double"); + + // NON-CANONICAL fallback: values outside the domain gate are only + // reachable via programmatic JSONValue construction; the formatter + // must still return SOMETHING rather than throw or emit garbage. + // Spelling is intentionally unpinned beyond being non-empty. + assert(canonicalizeJSON(JSONValue(5e-8)).length > 0, + "plain-decimal FAILED: out-of-domain fallback returned empty"); } // ── U+0000 rejection (contract change, external review) ── @@ -182,3 +225,162 @@ unittest assert(!hasDuplicateKeys(`{"k":"k","v":"k"}`), "duplicate-key FAILED: string VALUE matching a key wrongly flagged"); } + +// ── Number-domain enforcement (contract change) ── +// The check is LEXICAL on the raw token text: `100` and `1e2` parse to +// the same value but only the latter is rejected. Exponent notation is +// always unsupported; integers are digit-string-compared against 2^53 +// (never via double); fractions are magnitude-gated to [1e-6, 1e21). +unittest +{ + // Exponent notation — rejected regardless of value + assert(hasUnsupportedNumber(`{"x":1e2}`), + "number-domain FAILED: lowercase exponent not rejected"); + assert(hasUnsupportedNumber(`{"x":1E5}`), + "number-domain FAILED: uppercase exponent not rejected"); + assert(hasUnsupportedNumber(`{"x":1e-7}`), + "number-domain FAILED: negative exponent not rejected"); + assert(hasUnsupportedNumber(`[2.5e3]`), + "number-domain FAILED: exponent inside array not rejected"); + + // Same value without exponent notation — accepted + assert(!hasUnsupportedNumber(`{"x":100}`), + "number-domain FAILED: plain 100 wrongly rejected"); + + // Integer boundary: ±2^53 accepted, one past rejected — digit-string + // comparison, so the +1 cannot round back down to the boundary + assert(!hasUnsupportedNumber(`{"x":9007199254740992}`), + "number-domain FAILED: +2^53 wrongly rejected"); + assert(!hasUnsupportedNumber(`{"x":-9007199254740992}`), + "number-domain FAILED: -2^53 wrongly rejected"); + assert(hasUnsupportedNumber(`{"x":9007199254740993}`), + "number-domain FAILED: 2^53+1 not rejected"); + assert(hasUnsupportedNumber(`{"x":-9007199254740993}`), + "number-domain FAILED: -(2^53+1) not rejected"); + assert(hasUnsupportedNumber(`{"x":10000000000000000000}`), + "number-domain FAILED: 20-digit integer not rejected"); + + // Leading zeros don't inflate the digit count (parseJSON forbids + // them anyway; defensive against a laxer upstream parser) + assert(!hasUnsupportedNumber(`{"x":0}`), + "number-domain FAILED: zero wrongly rejected"); + + // Fraction magnitude gate: [1e-6, 1e21) accepted, outside rejected + assert(!hasUnsupportedNumber(`{"x":0.000001}`), + "number-domain FAILED: 1e-6 boundary fraction wrongly rejected"); + assert(hasUnsupportedNumber(`{"x":0.0000001}`), + "number-domain FAILED: sub-1e-6 fraction not rejected"); + assert(hasUnsupportedNumber(`{"x":1000000000000000000000.0}`), + "number-domain FAILED: 1e21 fraction not rejected"); + assert(!hasUnsupportedNumber(`{"x":0.5}`), + "number-domain FAILED: 0.5 wrongly rejected"); + assert(!hasUnsupportedNumber(`{"x":-0.0}`), + "number-domain FAILED: negative zero fraction wrongly rejected"); + + // Digits and 'e' inside STRING literals are not number tokens + assert(!hasUnsupportedNumber(`{"x":"1e2","1E5":"9007199254740993"}`), + "number-domain FAILED: number-like string content wrongly flagged"); +} + +// ── Negative zero canonicalizes to "0" (RFC 8785 / ES ToString) ── +// Any zero value — float or integer, signed or not — must emit exactly +// "0". Built with an explicit float-tagged -0.0 so the parser cannot +// demote it to integer zero first. +unittest +{ + JSONValue negZeroFloat = JSONValue(-0.0); + assert(canonicalizeJSON(negZeroFloat) == "0", + "negative-zero FAILED: float -0.0 did not emit \"0\""); + + auto negZeroDoc = parseJSON(`{"x":-0.0}`); + assert(canonicalizeJSON(negZeroDoc) == `{"x":0}`, + "negative-zero FAILED: parsed {\"x\":-0.0} did not emit {\"x\":0}"); + + // std.json parses the integer token -0 to integer zero, which the + // integer writer already emits as "0" — pinned here so a parser + // change cannot silently regress it. + auto negZeroInt = parseJSON(`{"x":-0}`); + assert(canonicalizeJSON(negZeroInt) == `{"x":0}`, + "negative-zero FAILED: parsed {\"x\":-0} did not emit {\"x\":0}"); + + // Positive zero unchanged + assert(canonicalizeJSON(JSONValue(0.0)) == "0", + "negative-zero FAILED: float +0.0 did not emit \"0\""); + + // Nonzero values must NOT be affected by the zero short-circuit + assert(canonicalizeJSON(JSONValue(0.1)) == "0.1", + "negative-zero FAILED: 0.1 formatting changed"); + assert(canonicalizeJSON(JSONValue(-1.5)) == "-1.5", + "negative-zero FAILED: -1.5 formatting changed"); +} + +// ── Single-document enforcement (contract change) ── +// Input must be exactly one complete JSON document with only whitespace +// around it. The scan runs on the RAW input because parseJSON stops at +// the end of the first complete value and silently ignores trailing +// data, concatenated documents and trailing commas, and parses empty +// input as a null value. +unittest +{ + import baionstd.types : StdError; + + // Exactly one document — accepted (null Nullable) + assert(scanSingleDocument(`{"a":1}`).isNull, + "single-document FAILED: plain object wrongly rejected"); + assert(scanSingleDocument(` {"a":1} `).isNull, + "single-document FAILED: surrounding whitespace wrongly rejected"); + assert(scanSingleDocument("{\"a\":1}\n").isNull, + "single-document FAILED: trailing newline wrongly rejected"); + assert(scanSingleDocument("\t\r\n [1,2] \t").isNull, + "single-document FAILED: mixed whitespace around array wrongly rejected"); + + // Bare scalars are single documents too + assert(scanSingleDocument(`"hi"`).isNull, + "single-document FAILED: bare string wrongly rejected"); + assert(scanSingleDocument(`true`).isNull, + "single-document FAILED: bare true wrongly rejected"); + assert(scanSingleDocument(`false `).isNull, + "single-document FAILED: bare false wrongly rejected"); + assert(scanSingleDocument(` null`).isNull, + "single-document FAILED: bare null wrongly rejected"); + assert(scanSingleDocument(`-12.5`).isNull, + "single-document FAILED: bare number wrongly rejected"); + + // Trailing data after the first document + assert(scanSingleDocument(`{"a":1} x`).get == StdError.trailingData, + "single-document FAILED: trailing garbage not rejected"); + assert(scanSingleDocument(`{"a":1}{"b":2}`).get == StdError.trailingData, + "single-document FAILED: concatenated documents not rejected"); + assert(scanSingleDocument(`"hi" "there"`).get == StdError.trailingData, + "single-document FAILED: two bare strings not rejected"); + assert(scanSingleDocument(`123 456`).get == StdError.trailingData, + "single-document FAILED: two bare numbers not rejected"); + assert(scanSingleDocument(`true false`).get == StdError.trailingData, + "single-document FAILED: two bare literals not rejected"); + assert(scanSingleDocument(`{"a":1},`).get == StdError.trailingData, + "single-document FAILED: comma after top-level document not rejected"); + + // Structural bytes inside STRING literals are not document ends + assert(scanSingleDocument(`{"a":"} x"}`).isNull, + "single-document FAILED: closer inside string mistaken for document end"); + + // Trailing comma — parseJSON swallows it, the contract rejects it + assert(scanSingleDocument(`{"a":1,}`).get == StdError.trailingComma, + "single-document FAILED: object trailing comma not rejected"); + assert(scanSingleDocument(`[1,2,]`).get == StdError.trailingComma, + "single-document FAILED: array trailing comma not rejected"); + assert(scanSingleDocument(`{"a":[1,]}`).get == StdError.trailingComma, + "single-document FAILED: nested trailing comma not rejected"); + assert(scanSingleDocument("{\"a\":1, \n}").get == StdError.trailingComma, + "single-document FAILED: whitespace-separated trailing comma not rejected"); + + // Empty and whitespace-only input + assert(scanSingleDocument(``).get == StdError.emptyInput, + "single-document FAILED: empty input not rejected"); + assert(scanSingleDocument(" \t\n").get == StdError.emptyInput, + "single-document FAILED: whitespace-only input not rejected"); + + // Commas between real members must NOT trip the trailing-comma check + assert(scanSingleDocument(`{"a":1,"b":[1,2]}`).isNull, + "single-document FAILED: legitimate commas wrongly flagged"); +} diff --git a/d/tools/canon_hash_main.d b/d/tools/canon_hash_main.d index e5c2994..79facf3 100644 --- a/d/tools/canon_hash_main.d +++ b/d/tools/canon_hash_main.d @@ -3,8 +3,10 @@ // // Reads UTF-8 JSON on stdin, writes the SHA-256 of its canonical form // (sorted keys, no whitespace, shortest-round-trip floats) as lowercase -// hex + newline. Exit 0 on success, nonzero on parse error or on a -// duplicate object key (decoded names) at any depth. +// hex + newline. Exit 0 on success, nonzero on parse error, on a +// duplicate object key (decoded names) at any depth, or when stdin is +// not exactly one JSON document (empty input, trailing data, +// concatenated documents, trailing comma). module tools.canon_hash_main; @@ -12,7 +14,8 @@ import std.array : appender; import std.json : parseJSON; import std.stdio : stdin, stdout, stderr; -import baionstd.canonical_json : canonicalizeJSON, hasDuplicateKeys; +import baionstd.canonical_json : canonicalizeJSON, hasDuplicateKeys, + hasUnsupportedNumber, scanSingleDocument; import baionstd.hash : sha256Hex; import baionstd.types : StdError, errorMessage; @@ -29,6 +32,19 @@ int main() // JSON and invalid UTF-8, which is exactly the CLI's error contract. auto j = parseJSON(cast(const(char)[]) buf[]); + // Single-document scan on the RAW input — parseJSON stops at the + // end of the first complete value and silently ignores trailing + // data, concatenated documents and trailing commas, and parses + // empty input as a null value. Runs FIRST among the raw scans: + // the other two assume the whole buffer is one well-formed + // document, which only this scan establishes. + auto docErr = scanSingleDocument(cast(const(char)[]) buf[]); + if (!docErr.isNull) + { + stderr.writeln("baion_canon_hash: ", errorMessage(docErr.get)); + return 1; + } + // Duplicate-key scan on the RAW input — parseJSON's associative // array has already deduplicated silently (keeps last), so the // check cannot run on the parsed value. Runs after parseJSON so @@ -39,6 +55,15 @@ int main() return 1; } + // Number-domain scan is also on the RAW input — parseJSON has + // already collapsed 1e2 to 100 and rounded out-of-range integers, + // so the lexical distinction only exists pre-parse. + if (hasUnsupportedNumber(cast(const(char)[]) buf[])) + { + stderr.writeln("baion_canon_hash: ", errorMessage(StdError.unsupportedNumber)); + return 1; + } + canonical = canonicalizeJSON(j); } catch (Exception e) diff --git a/go/canonical_json.go b/go/canonical_json.go index da69a39..ffecd48 100644 --- a/go/canonical_json.go +++ b/go/canonical_json.go @@ -36,6 +36,27 @@ var ErrEmbeddedNUL = errors.New("embedded U+0000 in string") // different values from the same bytes and silently break hash parity. var ErrDuplicateKey = errors.New("duplicate object key") +// ErrUnsupportedNumber rejects inputs containing a number token outside the +// cross-lineage numeric domain: exponent spellings, integers beyond ±2^53, +// and fractions outside [1e-6, 1e21). +// +// CROSS-LINEAGE CONTRACT: every language implementation uniformly REJECTS +// these tokens — float formatters across the seven lineages diverge exactly +// there (exponent notation, sub-1e-6 / ≥1e21 magnitudes, >2^53 precision +// loss), so accepting them would let two lineages emit different canonical +// bytes for the same input. The check is LEXICAL on the raw token spelling: +// `100` is in-domain while `1e2` is rejected even though the values are equal. +var ErrUnsupportedNumber = errors.New("unsupported number outside the cross-lineage numeric domain") + +// ErrLoneSurrogate rejects inputs containing an unpaired UTF-16 surrogate +// escape (\uD800–\uDFFF without its partner) inside a JSON string. +// +// CROSS-LINEAGE CONTRACT: every sibling lineage rejects lone surrogates at +// parse time; Go's decoder alone silently replaces them with U+FFFD, which +// would let Go produce a hash for input no sibling accepts. This must be +// detected on the RAW bytes — after decode the evidence is destroyed. +var ErrLoneSurrogate = errors.New("unsupported unpaired surrogate escape in string") + // CheckNoDuplicateKeys scans raw JSON input for duplicate object member names // at any depth and returns ErrDuplicateKey if one is found. // @@ -109,6 +130,147 @@ func CheckNoDuplicateKeys(data []byte) error { } } +// CheckNumberDomain scans raw JSON input and returns ErrUnsupportedNumber for +// any number token outside the cross-lineage domain: +// +// - exponent spellings (any 'e'/'E' in the token) — lexical, value-blind; +// - integer tokens (no '.') whose magnitude exceeds 9007199254740992 (2^53); +// - fraction tokens whose value v satisfies (v != 0 && |v| < 1e-6) || |v| >= 1e21. +// +// This check MUST run on the raw input bytes: json.Number preserves the raw +// token spelling (Decoder.Token honors UseNumber), which is the only place +// `100` and `1e2` are still distinguishable — the decoded value tree cannot +// make that distinction. +func CheckNumberDomain(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + // UseNumber makes Token() yield json.Number carrying the RAW token + // spelling — required for the lexical exponent check. + dec.UseNumber() + for { + tok, err := dec.Token() + if err == io.EOF { + return nil + } + if err != nil { + // Malformed JSON is not this check's verdict — the caller's + // decode pass reports syntax errors with proper diagnostics. + return nil + } + n, ok := tok.(json.Number) + if !ok { + continue + } + s := string(n) + if strings.ContainsAny(s, "eE") { + return ErrUnsupportedNumber + } + if !strings.Contains(s, ".") { + // Digit-string comparison, not ParseInt: stays exact for tokens + // wider than any machine integer. + if integerExceedsSafeRange(s) { + return ErrUnsupportedNumber + } + continue + } + // Fraction token: range-check the value. ParseFloat overflow returns + // ±Inf with ErrRange, which the >= 1e21 branch catches, so the error + // itself needs no separate handling. + v, _ := strconv.ParseFloat(s, 64) + if (v != 0 && math.Abs(v) < 1e-6) || math.Abs(v) >= 1e21 { + return ErrUnsupportedNumber + } + } +} + +// integerExceedsSafeRange reports whether an exponent-free integer token's +// magnitude exceeds 2^53 (9007199254740992), comparing digit strings so +// arbitrarily wide tokens are judged exactly. +func integerExceedsSafeRange(s string) bool { + digits := strings.TrimPrefix(s, "-") + digits = strings.TrimLeft(digits, "0") + if digits == "" { + return false // zero + } + const maxSafe = "9007199254740992" // 2^53 + if len(digits) != len(maxSafe) { + return len(digits) > len(maxSafe) + } + return digits > maxSafe +} + +// CheckNoLoneSurrogates scans raw JSON input for unpaired UTF-16 surrogate +// escapes and returns ErrLoneSurrogate if one is found: a \uD800–\uDBFF escape +// not immediately followed by an escaped \uDC00–\uDFFF, or a \uDC00–\uDFFF +// escape with no preceding high half. +// +// Backslash-run parity decides whether a `\u` is an active escape: an escape +// starts only on the odd trailing backslash of a run, so `\\ud800` (a literal +// backslash followed by the text "ud800") is NOT a surrogate and passes. +// Escapes only occur inside strings in well-formed JSON; a stray backslash +// elsewhere is a syntax error the decode pass rejects anyway, so scanning the +// whole input is safe. +func CheckNoLoneSurrogates(data []byte) error { + for i := 0; i < len(data); { + if data[i] != '\\' { + i++ + continue + } + j := i + for j < len(data) && data[j] == '\\' { + j++ + } + run := j - i + i = j + if run%2 == 0 { + continue // backslashes pair off into literal backslashes + } + // The trailing odd backslash escapes data[j]. + if j >= len(data) || data[j] != 'u' { + i = j + 1 + continue + } + cp, ok := hex4(data, j+1) + if !ok { + // Malformed \u escape — the decode pass reports it. + i = j + 1 + continue + } + switch { + case cp >= 0xD800 && cp <= 0xDBFF: + // High surrogate: valid only when immediately followed by an + // escaped low surrogate. + paired := false + if j+11 <= len(data) && data[j+5] == '\\' && data[j+6] == 'u' { + if lo, ok2 := hex4(data, j+7); ok2 && lo >= 0xDC00 && lo <= 0xDFFF { + paired = true + } + } + if !paired { + return ErrLoneSurrogate + } + i = j + 11 // consume the pair so its low half is not re-seen as lone + case cp >= 0xDC00 && cp <= 0xDFFF: + // Low surrogate reached without a consuming high half above. + return ErrLoneSurrogate + default: + i = j + 5 + } + } + return nil +} + +// hex4 parses four hex digits at data[pos:pos+4] as a code unit. +func hex4(data []byte, pos int) (uint32, bool) { + if pos+4 > len(data) { + return 0, false + } + v, err := strconv.ParseUint(string(data[pos:pos+4]), 16, 32) + if err != nil { + return 0, false + } + return uint32(v), true +} + // CanonicalizeJSON converts any JSON-compatible value to canonical JSON string. func CanonicalizeJSON(v interface{}) string { var b strings.Builder @@ -278,6 +440,13 @@ func formatFloat64(v float64) string { if math.IsInf(v, -1) { return "null" } + // CROSS-LINEAGE CONTRACT: any zero emits exactly "0" (RFC 8785 §3.2.2.3 / + // ES ToString). -0.0 == 0 in Go, so this also catches negative zero, which + // FormatFloat would otherwise spell "-0" and break parity with the five + // lineages that emit "0". + if v == 0 { + return "0" + } return strconv.FormatFloat(v, 'f', -1, 64) } diff --git a/go/canonical_json_test.go b/go/canonical_json_test.go index 767eddb..64b26c0 100644 --- a/go/canonical_json_test.go +++ b/go/canonical_json_test.go @@ -2,6 +2,7 @@ package baionstd import ( "encoding/json" + "math" "strings" "testing" ) @@ -214,6 +215,129 @@ func TestCanonicalJSON_DuplicateKeyRejection(t *testing.T) { } } +// Number-domain rejection contract: the check is lexical on the raw token — +// `100` is in-domain, `1E2` is rejected even though the values are equal. +// Domain: no exponent spelling, integers within ±2^53, fractions in [1e-6, 1e21). +func TestCanonicalJSON_NumberDomainRejection(t *testing.T) { + rejects := []struct { + name string + input string + }{ + {"exponent_lower", `{"x":1e5}`}, + {"exponent_upper", `{"x":1E2}`}, + {"exponent_large", `{"x":1e21}`}, + {"exponent_negative", `{"x":1e-7}`}, + {"exponent_overflow", `{"x":1e400}`}, + {"exponent_bare", `1e0`}, + {"int_beyond_2p53", `{"x":9007199254740993}`}, + {"int_beyond_2p53_negative", `{"x":-9007199254740993}`}, + {"int_much_wider_than_int64", `{"x":99999999999999999999999999}`}, + {"fraction_below_1e_minus_6", `{"x":0.0000001}`}, + {"fraction_at_or_above_1e21", `{"x":1000000000000000000000.5}`}, + {"nested_in_array", `{"a":[1,2,1e2]}`}, + } + for _, tt := range rejects { + t.Run("reject_"+tt.name, func(t *testing.T) { + if err := CheckNumberDomain([]byte(tt.input)); err != ErrUnsupportedNumber { + t.Errorf("input %q: want ErrUnsupportedNumber, got %v", tt.input, err) + } + }) + } + + accepts := []struct { + name string + input string + }{ + {"plain_int", `{"x":100}`}, + {"max_safe_int", `{"x":9007199254740992}`}, + {"min_safe_int", `{"x":-9007199254740992}`}, + {"fraction_at_1e_minus_6", `{"x":0.000001}`}, + {"fraction_plain", `{"x":0.1}`}, + {"negative_zero_fraction", `{"x":-0.0}`}, + {"zero", `{"x":0}`}, + {"bare_fraction", `1.5`}, + } + for _, tt := range accepts { + t.Run("accept_"+tt.name, func(t *testing.T) { + if err := CheckNumberDomain([]byte(tt.input)); err != nil { + t.Errorf("input %q: want nil, got %v", tt.input, err) + } + }) + } +} + +// Lone-surrogate rejection contract: an unpaired \uD800–\uDFFF escape in the +// raw input is rejected; a literal backslash followed by the text "ud800" +// (backslash-run parity even) and properly paired surrogates pass. Detection +// must be on raw bytes — Go's decoder replaces lone surrogates with U+FFFD. +func TestCanonicalJSON_LoneSurrogateRejection(t *testing.T) { + rejects := []struct { + name string + input string + }{ + {"lone_high", `{"x":"\ud800"}`}, + {"lone_high_upper", `{"x":"\uD800"}`}, + {"lone_low", `{"x":"\udc00"}`}, + {"high_then_non_surrogate_escape", `{"x":"\ud800\n"}`}, + {"high_then_literal_text", `{"x":"\ud800abc"}`}, + {"high_at_end_of_input", `{"x":"\ud800`}, + {"literal_backslash_then_lone_high", `{"x":"\\\ud800"}`}, // run of 3: literal backslash + active lone escape + {"low_before_high", `{"x":"\udc00\ud800"}`}, + } + for _, tt := range rejects { + t.Run("reject_"+tt.name, func(t *testing.T) { + if err := CheckNoLoneSurrogates([]byte(tt.input)); err != ErrLoneSurrogate { + t.Errorf("input %q: want ErrLoneSurrogate, got %v", tt.input, err) + } + }) + } + + accepts := []struct { + name string + input string + }{ + {"literal_backslash_ud800", `{"x":"\\ud800"}`}, // even run: literal backslash + text + {"double_literal_backslash_ud800", `{"x":"\\\\ud800"}`}, + {"paired_surrogates", `{"x":"\ud83d\ude00"}`}, + {"paired_surrogates_upper", `{"x":"\uD83D\uDE00"}`}, + {"raw_utf8_emoji", `{"x":"` + "\U0001F600" + `"}`}, + {"non_surrogate_escape", `{"x":"\u0041"}`}, + {"plain_string", `{"x":"hello"}`}, + } + for _, tt := range accepts { + t.Run("accept_"+tt.name, func(t *testing.T) { + if err := CheckNoLoneSurrogates([]byte(tt.input)); err != nil { + t.Errorf("input %q: want nil, got %v", tt.input, err) + } + }) + } +} + +// Negative-zero contract: any zero value emits exactly "0" (RFC 8785 / ES +// ToString) — both the fraction spelling -0.0 (float path) and the integer +// spelling -0 (int64 path). +func TestCanonicalJSON_NegativeZero(t *testing.T) { + tests := []struct { + name string + val interface{} + want string + }{ + {"neg_zero_fraction_token", json.Number("-0.0"), "0"}, + {"neg_zero_int_token", json.Number("-0"), "0"}, + {"pos_zero_fraction_token", json.Number("0.0"), "0"}, + {"neg_zero_float64", math.Copysign(0, -1), "0"}, + {"in_object", map[string]interface{}{"x": json.Number("-0.0")}, `{"x":0}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanonicalizeJSON(tt.val) + if got != tt.want { + t.Errorf("value %v (%T)\nwant: %s\ngot: %s", tt.val, tt.val, tt.want, got) + } + }) + } +} + func TestCanonicalJSON_EmptyObject(t *testing.T) { got := CanonicalizeJSON(map[string]interface{}{}) if got != "{}" { diff --git a/go/cmd/baion_canon_hash/main.go b/go/cmd/baion_canon_hash/main.go index 98803a7..7a3fa56 100644 --- a/go/cmd/baion_canon_hash/main.go +++ b/go/cmd/baion_canon_hash/main.go @@ -34,6 +34,21 @@ func main() { os.Exit(1) } + // Number-domain enforcement is LEXICAL (`100` in-domain, `1e2` rejected), + // so it too needs the raw bytes — the decoded tree loses the spelling. + if err := baionstd.CheckNumberDomain(input); err != nil { + fmt.Fprintf(os.Stderr, "baion_canon_hash: reject: unsupported number in input (%v)\n", err) + os.Exit(1) + } + + // Lone-surrogate detection needs the raw bytes as well: Go's decoder + // silently replaces unpaired surrogates with U+FFFD, destroying the + // evidence every sibling lineage rejects on. + if err := baionstd.CheckNoLoneSurrogates(input); err != nil { + fmt.Fprintf(os.Stderr, "baion_canon_hash: reject: unpaired surrogate escape in input (%v)\n", err) + os.Exit(1) + } + // json.Number preserves number spelling so canonicalization — not the // decoder's float64 round-trip — decides the canonical numeric form. dec := json.NewDecoder(bytes.NewReader(input)) diff --git a/haskell/app/Main.hs b/haskell/app/Main.hs index 3489083..6f2e2cf 100644 --- a/haskell/app/Main.hs +++ b/haskell/app/Main.hs @@ -6,7 +6,11 @@ -- to stderr with a nonzero exit so pipelines fail loudly. module Main (main) where -import Baion.STD.CanonicalJson (canonicalizeJsonChecked, checkNoDuplicateKeys) +import Baion.STD.CanonicalJson + ( canonicalizeJsonChecked, + checkNoDuplicateKeys, + checkNumberDomain, + ) import Baion.STD.Hash (sha256HexBytes) import qualified Data.Aeson as A import qualified Data.ByteString as BS @@ -18,17 +22,26 @@ import System.IO (hPutStrLn, stderr) main :: IO () main = do input <- BS.getContents + -- STRICT single-document contract: aeson >= 2.2's eitherDecodeStrict' + -- (Data.Aeson.Decoding path) requires exactly one complete JSON + -- document with the whole input consumed — trailing garbage, a second + -- document, a trailing comma, and empty input all fail here (only + -- trailing whitespace is allowed). Suite tests 15-19 pin this so an + -- aeson behavior change cannot silently relax it. case A.eitherDecodeStrict' input :: Either String A.Value of Left err -> do hPutStrLn stderr ("baion-canon-hash: parse error: " ++ err) exitFailure Right v -> - -- Duplicate-key check runs on the RAW bytes (aeson's KeyMap - -- drops duplicates at parse, so the decoded Value can't tell); - -- then checked canonicalization rejects any string containing - -- U+0000 (aeson preserves NUL in Text, so the CLI would - -- otherwise pass it through to the digest). - case checkNoDuplicateKeys input >> canonicalizeJsonChecked v of + -- Duplicate-key and number-domain checks both run on the RAW + -- bytes (aeson's KeyMap drops duplicates at parse, and Scientific + -- erases the lexical 100-vs-1e2 distinction the number contract + -- is defined over); then checked canonicalization rejects any + -- string containing U+0000 (aeson preserves NUL in Text, so the + -- CLI would otherwise pass it through to the digest). + case checkNoDuplicateKeys input + >> checkNumberDomain input + >> canonicalizeJsonChecked v of Left err -> do hPutStrLn stderr ("baion-canon-hash: " ++ err) exitFailure diff --git a/haskell/baionstd-public.cabal b/haskell/baionstd-public.cabal index 2759e9f..98c068b 100644 --- a/haskell/baionstd-public.cabal +++ b/haskell/baionstd-public.cabal @@ -1,6 +1,6 @@ cabal-version: 3.0 name: baionstd-public -version: 0.1.2 +version: 0.2.0 synopsis: BAION canonical JSON + SHA-256 (Haskell) license: MIT license-file: LICENSE diff --git a/haskell/src/Baion/STD/CanonicalJson.hs b/haskell/src/Baion/STD/CanonicalJson.hs index 3daf405..0dccd81 100644 --- a/haskell/src/Baion/STD/CanonicalJson.hs +++ b/haskell/src/Baion/STD/CanonicalJson.hs @@ -2,12 +2,13 @@ -- | BAION canonical JSON for Haskell — public standalone library. -- Deterministic canonicalization of arbitrary JSON values --- (RFC 8785-style: sorted object keys, minimal escapes, shortest --- round-tripping number formatting). +-- (RFC 8785-style: sorted object keys, minimal escapes, ECMAScript +-- ToString plain-decimal number formatting over double semantics). module Baion.STD.CanonicalJson ( canonicalizeJson, canonicalizeJsonChecked, checkNoDuplicateKeys, + checkNumberDomain, writeJsonString, ) where @@ -22,12 +23,13 @@ import Data.Aeson.Decoding.Tokens import qualified Data.Aeson.Key as AK import qualified Data.Aeson.KeyMap as AKM import qualified Data.ByteString as BS -import Data.Char (ord) +import Data.Char (intToDigit, ord) import qualified Data.Map.Strict as Map import qualified Data.Scientific as S import qualified Data.Set as Set import qualified Data.Text as T import qualified Data.Vector as V +import Numeric (floatToDigits) import Text.Printf (printf) canonicalizeJson :: A.Value -> String @@ -92,6 +94,96 @@ checkNoDuplicateKeys bs = case scanTokens (bsToTokens bs) of scanRecord _ (TkRecordEnd k) = Right (Just k) scanRecord _ (TkRecordErr _) = Right Nothing +-- | Reject number tokens outside the cross-lineage number domain. +-- This must scan the RAW input: aeson decodes every number spelling +-- into 'S.Scientific', so by the time a 'A.Value' exists the lexical +-- distinction between @100@ and @1e2@ is gone — and the contract is +-- lexical (exponent SPELLING is rejected even when the value is safe). +-- Sibling of 'checkNoDuplicateKeys': a linear pass over the raw bytes +-- that skips string literals escape-aware; outside strings, a digit or +-- @-@ can only begin a number token in well-formed JSON (the CLI runs +-- this only after a successful strict decode), so the maximal run of +-- number-token bytes IS the token. +-- CROSS-LINEAGE CONTRACT: all 7 lineages reject identically: +-- * any exponent spelling (e/E), regardless of value; +-- * integer tokens (no @.@) beyond +/-9007199254740992, compared as +-- digit strings so 9007199254740993 is caught despite rounding to +-- a representable Double; +-- * fraction tokens whose Double value v has +-- (v /= 0 && abs v < 1e-6) || abs v >= 1e21. +checkNumberDomain :: BS.ByteString -> Either String () +checkNumberDomain = goTop + where + goTop bs = case BS.uncons bs of + Nothing -> Right () + Just (c, rest) + | c == 0x22 -> goTop (skipString rest) -- '"' + | isNumStart c -> + let (tok, rest') = BS.span isNumByte bs + in checkToken (map (toEnum . fromIntegral) (BS.unpack tok)) + >> goTop rest' + | otherwise -> goTop rest + + isNumStart c = c == 0x2d || (c >= 0x30 && c <= 0x39) -- '-' / digit + isNumByte c = + (c >= 0x30 && c <= 0x39) -- digit + || c == 0x2d -- '-' + || c == 0x2b -- '+' + || c == 0x2e -- '.' + || c == 0x65 -- 'e' + || c == 0x45 -- 'E' + + -- Inside a string literal: any backslash consumes the next byte + -- (enough to keep \" from ending the scan; multi-byte UTF-8 never + -- contains 0x22/0x5c continuation bytes, so byte-wise is safe). + skipString bs = case BS.uncons bs of + Nothing -> BS.empty + Just (c, rest) + | c == 0x5c -> skipString (BS.drop 1 rest) -- '\\' + | c == 0x22 -> rest -- closing '"' + | otherwise -> skipString rest + + checkToken :: String -> Either String () + checkToken s + | any (\c -> c == 'e' || c == 'E') s = + Left + ( "unsupported number " + ++ show s + ++ ": scientific (exponent) notation is outside the" + ++ " cross-lineage number contract" + ) + | '.' `elem` s = case reads s :: [(Double, String)] of + [(v, "")] + | (v /= 0 && abs v < 1e-6) || abs v >= 1e21 -> + Left + ( "unsupported number " + ++ show s + ++ ": fraction magnitude outside the cross-lineage" + ++ " range [1e-6, 1e21)" + ) + | otherwise -> Right () + -- Unreachable after a successful strict decode; reject rather + -- than let an unparseable spelling reach the digest. + _ -> Left ("unsupported number " ++ show s ++ ": unparseable token") + | otherwise = + -- Integer token: digit-string comparison, never floats — + -- 9007199254740993 rounds to a representable Double and + -- would slip through a numeric compare. + let digits = dropWhile (== '-') s + maxSafe = "9007199254740992" + beyond = + length digits > length maxSafe + || (length digits == length maxSafe && digits > maxSafe) + in if beyond + then + Left + ( "unsupported number " + ++ show s + ++ ": integer beyond +/-9007199254740992" + ++ " (cross-lineage safe-integer bound)" + ) + else Right () + -- | Recursive walk: does any string (key or value) contain U+0000? containsNul :: A.Value -> Bool containsNul (A.String t) = T.any (== '\NUL') t @@ -127,33 +219,51 @@ canonicalizeValue (A.Object obj) = ] ++ "}" --- | Show a Double in canonical-JSON form per RFC 8785 / ECMA-262 --- §7.1.12.1. Integer-valued doubles emit without trailing decimal. --- Fractional doubles use the shortest precision that round-trips --- exactly (mirrors the D lineage's formatShortest's %g shortest-search loop; --- prevents printf "%.17g" from padding 1.5 → "1.50000000000000000", --- which was the pre-2026-05-04 Haskell divergence). +-- | Show a Double in canonical-JSON form per ECMA-262 §7.1.12.1 +-- (Number::toString radix 10), plain decimal only — the reference +-- rendering shared by the C/Go/OCaml lineages. The previous %g +-- shortest-search loop switched to scientific notation below 0.01 +-- and near 1e21 ("1.0e-3"), which diverged from the reference for +-- every fraction in those bands (fixed 2026-07-15). +-- 'Numeric.floatToDigits' 10 yields exactly the minimal (shortest +-- round-tripping) digit string ES-262 specifies, with no trailing +-- zeros, so reassembly is pure positional bookkeeping. +-- CROSS-LINEAGE CONTRACT: byte-identical to ECMAScript ToString for +-- doubles inside the gated fraction domain [1e-6, 1e21). showDouble :: Double -> String showDouble d | isNaN d || isInfinite d = "null" - | d == fromInteger (round d) && abs d < 1e15 = - show (round d :: Integer) - | otherwise = shortestRoundTrip d - --- | Search for the smallest precision whose %g formatting round-trips --- back to the original Double. CROSS-LINEAGE CONTRACT: behavior must --- match formatShortest in `d/source/baionstd/canonical_json.d`. -shortestRoundTrip :: Double -> String -shortestRoundTrip d = go (1 :: Int) + | d == 0 = "0" -- ES-262: ToString of +0 AND -0 is "0" + | d < 0 = '-' : positiveToString (negate d) + | otherwise = positiveToString d + +-- ES-262 notation: value = D × 10^(n − k) with D the minimal digit +-- string and k = length D. floatToDigits 10 x = (digits, e) means +-- x = 0.D × 10^e = D × 10^(e − k), so n = e directly. +positiveToString :: Double -> String +positiveToString d = + let (ds, n) = floatToDigits 10 d + in assembleEs262 (map intToDigit ds) n + +assembleEs262 :: String -> Int -> String +assembleEs262 ds n + | k <= n && n <= 21 = ds ++ replicate (n - k) '0' + | 0 < n && n <= k = take n ds ++ "." ++ drop n ds + | (-5) <= n && n <= 0 = "0." ++ replicate (negate n) '0' ++ ds + -- NON-CANONICAL defensive fallback: the number-domain gate pins + -- |v| inside [1e-6, 1e21), so ES-262's exponent branches are + -- unreachable through the CLI; emit the ES-262 exponent form + -- rather than crash if a library caller bypasses the gate. + | otherwise = + let mantissa = case ds of + [c] -> [c] + (c : rest) -> c : '.' : rest + [] -> "0" + e = n - 1 + sign = if e >= 0 then "+" else "" + in mantissa ++ "e" ++ sign ++ show e where - go p - | p > 17 = printf "%.17g" d -- fallback (unreachable for finite IEEE-754) - | (readMaybe (printf "%.*g" p d) :: Maybe Double) == Just d = - printf "%.*g" p d - | otherwise = go (p + 1) - readMaybe s = case reads s of - [(x, "")] -> Just x - _ -> Nothing + k = length ds commaJoin :: [String] -> String commaJoin [] = "" diff --git a/haskell/test/ConformanceTest.hs b/haskell/test/ConformanceTest.hs index 59a39f9..805baa5 100644 --- a/haskell/test/ConformanceTest.hs +++ b/haskell/test/ConformanceTest.hs @@ -90,9 +90,111 @@ conformanceTests = testCase "Test 13: distinct keys sharing a prefix accepted" $ assertNoDuplicate "{\"aa\":1,\"ab\":2}", testCase "Test 14: repeated array values are not duplicate keys" $ - assertNoDuplicate "{\"b\":1,\"a\":[1,2]}" + assertNoDuplicate "{\"b\":1,\"a\":[1,2]}", + testCase "Test 15: strict decode rejects trailing garbage" $ + assertDecodeRejected "{\"a\":1} x", + testCase "Test 16: strict decode rejects a second document" $ + assertDecodeRejected "{\"a\":1}{\"b\":2}", + testCase "Test 17: strict decode rejects a trailing comma" $ + assertDecodeRejected "{\"a\":1,}", + testCase "Test 18: strict decode rejects empty input" $ + assertDecodeRejected "", + testCase "Test 19: strict decode allows trailing whitespace" $ + assertDecodeAccepted "{\"a\":1} \n\t", + testCase "Test 20: exponent-spelled number rejected (1E2 vs 100)" $ do + assertNumberRejected "{\"x\":1E2}" + assertNumberRejected "{\"x\":1e5}" + assertNumberAccepted "{\"x\":100}", + testCase "Test 21: exponent spelling rejected regardless of value" $ do + assertNumberRejected "{\"x\":1e21}" + assertNumberRejected "{\"x\":1e-7}" + assertNumberRejected "{\"x\":1e400}", + testCase "Test 22: integer beyond +9007199254740992 rejected" $ do + assertNumberRejected "{\"x\":9007199254740993}" + assertNumberAccepted "{\"x\":9007199254740992}", + testCase "Test 23: integer beyond -9007199254740992 rejected" $ do + assertNumberRejected "{\"x\":-9007199254740993}" + assertNumberAccepted "{\"x\":-9007199254740992}", + testCase "Test 24: fraction below 1e-6 rejected, boundary kept" $ do + assertNumberRejected "{\"x\":0.0000001}" + assertNumberAccepted "{\"x\":0.000001}" + assertNumberAccepted "{\"x\":0.1}" + assertNumberAccepted "{\"x\":-0.0}", + testCase "Test 25: exponent-like spellings inside strings pass" $ do + assertNumberAccepted "{\"x\":\"1e21\"}" + assertNumberAccepted "{\"x\":\"a\\\"1E5\",\"y\":100}", + testCase "Test 26: sub-0.01 fractions emit plain decimal, never exponent" $ do + assertCanonicalizes "{\"x\":0.001}" "{\"x\":0.001}" + assertCanonicalizes "{\"x\":0.0001}" "{\"x\":0.0001}" + assertCanonicalizes "{\"x\":0.000123}" "{\"x\":0.000123}" + assertCanonicalizes "{\"x\":0.00001}" "{\"x\":0.00001}" + assertCanonicalizes "{\"x\":0.000001}" "{\"x\":0.000001}", + testCase "Test 27: fraction near 1e21 collapses to 21-digit integer via double" $ + assertCanonicalizes + "{\"x\":100000000000000000000.5}" + "{\"x\":100000000000000000000}", + testCase "Test 28: integer tokens keep plain-integer printing" $ do + assertCanonicalizes "{\"x\":9007199254740992}" "{\"x\":9007199254740992}" + assertCanonicalizes "{\"x\":-42}" "{\"x\":-42}" + assertCanonicalizes "{\"x\":0}" "{\"x\":0}", + testCase "Test 29: signed zero and mid-range fractions unchanged" $ do + assertCanonicalizes "{\"x\":-0.0}" "{\"x\":0}" + assertCanonicalizes "{\"x\":-0.375}" "{\"x\":-0.375}" + assertCanonicalizes "{\"x\":0.1}" "{\"x\":0.1}" + assertCanonicalizes "{\"x\":123.456}" "{\"x\":123.456}" + assertCanonicalizes "{\"x\":1.5}" "{\"x\":1.5}" + assertCanonicalizes "{\"x\":1.0}" "{\"x\":1}" ] +-- STRICT single-document contract (tests 15-19): aeson >= 2.2's +-- eitherDecodeStrict' must consume the whole input as exactly one +-- JSON document (trailing whitespace only). Pinned here so an aeson +-- upgrade cannot silently relax the CLI's parse discipline. +assertDecodeRejected :: String -> Assertion +assertDecodeRejected raw = + case A.eitherDecodeStrict' (BSC.pack raw) :: Either String A.Value of + Left _ -> return () + Right v -> + assertFailure + ("expected strict-decode rejection for " ++ show raw ++ ", got: " ++ show v) + +assertDecodeAccepted :: String -> Assertion +assertDecodeAccepted raw = + case A.eitherDecodeStrict' (BSC.pack raw) :: Either String A.Value of + Left err -> assertFailure ("expected acceptance for " ++ show raw ++ ", got: " ++ err) + Right _ -> return () + +-- Number rendering contract (tests 26-29): canonical output must be +-- ECMAScript-ToString plain decimal over DOUBLE semantics, matching +-- the C/Go/OCaml reference lineages byte-for-byte. Sub-0.01 fractions +-- are where the old %g formatter drifted into exponent notation. +assertCanonicalizes :: String -> String -> Assertion +assertCanonicalizes raw expected = + case A.eitherDecodeStrict' (BSC.pack raw) :: Either String A.Value of + Left err -> assertFailure ("fixture must parse: " ++ err) + Right v -> canonicalizeJson v @?= expected + +-- Number-domain contract (tests 20-25): lexical check over raw bytes, +-- error text must carry both "unsupported" and "number" so pipeline +-- greps can classify the failure. +assertNumberRejected :: String -> Assertion +assertNumberRejected raw = + case checkNumberDomain (BSC.pack raw) of + Left err -> + assertBool + ("error message must mention unsupported number, got: " ++ err) + ( T.isInfixOf "unsupported" (T.pack err) + && T.isInfixOf "number" (T.pack err) + ) + Right () -> + assertFailure ("expected number-domain rejection for: " ++ raw) + +assertNumberAccepted :: String -> Assertion +assertNumberAccepted raw = + case checkNumberDomain (BSC.pack raw) of + Left err -> assertFailure ("expected acceptance for " ++ raw ++ ", got: " ++ err) + Right () -> return () + testSha256Vector :: Assertion testSha256Vector = do ref_ <- loadReference diff --git a/ocaml/cli/baion_canon_hash.ml b/ocaml/cli/baion_canon_hash.ml index 2149b42..12ced11 100644 --- a/ocaml/cli/baion_canon_hash.ml +++ b/ocaml/cli/baion_canon_hash.ml @@ -7,6 +7,22 @@ let () = let input = In_channel.input_all In_channel.stdin in + (* Number-domain enforcement runs on the RAW bytes before parsing: + yojson normalizes 1e2 to the same value as 100, so exponent + spelling is only visible lexically. *) + (match Baionstd_public.Canonical_json.reject_unsupported_numbers input with + | exception Baionstd_public.Canonical_json.Number_rejected msg -> + prerr_endline ("baion_canon_hash: invalid input: " ^ msg); + exit 1 + | () -> ()); + (* Lone-surrogate enforcement also runs on the RAW bytes: yojson maps + a lone backslash-udc00 escape to U+FFFD by parse time, so the + unpaired spelling is only visible lexically. *) + (match Baionstd_public.Canonical_json.check_no_lone_surrogates input with + | exception Baionstd_public.Canonical_json.Lone_surrogate msg -> + prerr_endline ("baion_canon_hash: invalid input: " ^ msg); + exit 1 + | () -> ()); match Yojson.Safe.from_string input with | exception Yojson.Json_error msg -> prerr_endline ("baion_canon_hash: JSON parse error: " ^ msg); diff --git a/ocaml/lib/canonical_json.ml b/ocaml/lib/canonical_json.ml index 65e6e50..3ef9fa4 100644 --- a/ocaml/lib/canonical_json.ml +++ b/ocaml/lib/canonical_json.ml @@ -31,6 +31,178 @@ let rec reject_nul (v : Yojson.Safe.t) = (** Raised when an object carries the same member name twice (any depth). *) exception Duplicate_key of string +(** Raised when the raw input carries a number outside the canonical + domain: exponent notation, an integer beyond 2^53, or a fraction + outside [1e-6, 1e21). *) +exception Number_rejected of string + +(* 2^53 — the largest integer every lineage can represent exactly. *) +let max_safe_integer_digits = "9007199254740992" + +(* Reject one raw number token. Integer comparison is done on the digit + string (never through a float) because 9007199254740993 rounds to + 9007199254740992.0 and would slip past a float compare. *) +let check_number_token tok = + if String.exists (fun c -> c = 'e' || c = 'E') tok then + raise + (Number_rejected ("unsupported number (exponent notation): " ^ tok)) + else if String.contains tok '.' then ( + match float_of_string_opt tok with + | None -> () (* malformed token: leave it to the JSON parser's error *) + | Some v -> + let a = Float.abs v in + if (v <> 0. && a < 1e-6) || a >= 1e21 then + raise + (Number_rejected + ("unsupported number (fraction out of canonical range): " + ^ tok))) + else + let digits = + if tok.[0] = '-' then String.sub tok 1 (String.length tok - 1) else tok + in + (* JSON forbids leading zeros, but strip them defensively so a + malformed "0009" never inflates the length compare below. *) + let digits = + let dl = String.length digits in + let j = ref 0 in + while !j < dl - 1 && digits.[!j] = '0' do + incr j + done; + String.sub digits !j (dl - !j) + in + let dl = String.length digits in + let ml = String.length max_safe_integer_digits in + if dl > ml || (dl = ml && String.compare digits max_safe_integer_digits > 0) + then + raise + (Number_rejected ("unsupported number (integer exceeds 2^53): " ^ tok)) + +(* CROSS-LINEAGE CONTRACT: number-domain enforcement is a LEXICAL pass + over the raw input text. yojson parses 1e2 and 100 to the same value, + so by parse time the exponent spelling is unrecoverable — the scan + must happen on the bytes. Same skip-strings technique as the D + lineage's hasDuplicateKeys: walk the raw text, skip string literals + (honoring backslash escapes so an escaped quote never ends the + string early), and inspect every number token. 'e' inside true/false + never trips this because tokens only start at a digit or '-'. *) +let reject_unsupported_numbers (raw : string) : unit = + let len = String.length raw in + let i = ref 0 in + while !i < len do + let c = raw.[!i] in + if c = '"' then begin + incr i; + let closed = ref false in + while (not !closed) && !i < len do + match raw.[!i] with + | '\\' -> i := !i + 2 + | '"' -> + closed := true; + incr i + | _ -> incr i + done + end + else if c = '-' || (c >= '0' && c <= '9') then begin + let start = !i in + while + !i < len + && + match raw.[!i] with + | '0' .. '9' | '-' | '+' | '.' | 'e' | 'E' -> true + | _ -> false + do + incr i + done; + check_number_token (String.sub raw start (!i - start)) + end + else incr i + done + +(** Raised when the raw input carries an unpaired UTF-16 surrogate escape. *) +exception Lone_surrogate of string + +(* Parse four hex digits at raw.[pos..pos+3] as a UTF-16 code unit. + Returns None on truncation or a non-hex digit — a malformed \u escape + is the JSON parser's error to report, not ours. *) +let hex4 raw pos = + let len = String.length raw in + if pos + 4 > len then None + else + let v = ref 0 in + let ok = ref true in + for k = pos to pos + 3 do + let d = + match raw.[k] with + | '0' .. '9' as c -> Char.code c - Char.code '0' + | 'a' .. 'f' as c -> Char.code c - Char.code 'a' + 10 + | 'A' .. 'F' as c -> Char.code c - Char.code 'A' + 10 + | _ -> + ok := false; + 0 + in + v := (!v * 16) + d + done; + if !ok then Some !v else None + +(* CROSS-LINEAGE CONTRACT: unpaired UTF-16 surrogate escapes must be + rejected by all seven lineage libraries. This is a LEXICAL pass over + the raw input (mirrors Go's CheckNoLoneSurrogates): yojson maps a + lone \udc00 to U+FFFD by parse time, indistinguishable from a + genuine U+FFFD in the input, so the scan must happen on the bytes. + (The lone HIGH half \ud800 yojson already refuses at parse; the + lone LOW half is the gap this closes.) + + Backslash-run parity decides whether a \u is an ACTIVE escape: an + escape starts only on the odd trailing backslash of a run, so + \\udc00 (a literal backslash + the text "udc00") is not a surrogate + and passes. Escapes only occur inside strings in well-formed JSON; + a stray backslash elsewhere is a syntax error the parse pass rejects + anyway, so scanning the whole input is safe. *) +let check_no_lone_surrogates (raw : string) : unit = + let len = String.length raw in + let i = ref 0 in + while !i < len do + if raw.[!i] <> '\\' then incr i + else begin + let j = ref !i in + while !j < len && raw.[!j] = '\\' do + incr j + done; + let run = !j - !i in + i := !j; + if run mod 2 = 1 then + if !j < len && raw.[!j] = 'u' then begin + match hex4 raw (!j + 1) with + | None -> i := !j + 1 (* malformed escape: parser reports it *) + | Some cp when cp >= 0xD800 && cp <= 0xDBFF -> + (* High surrogate: valid only when immediately followed by + an escaped low surrogate; consume the pair so its low + half is never re-seen as lone. *) + let paired = + !j + 11 <= len + && raw.[!j + 5] = '\\' + && raw.[!j + 6] = 'u' + && + match hex4 raw (!j + 7) with + | Some lo -> lo >= 0xDC00 && lo <= 0xDFFF + | None -> false + in + if not paired then + raise + (Lone_surrogate + "unsupported unpaired surrogate escape in string"); + i := !j + 11 + | Some cp when cp >= 0xDC00 && cp <= 0xDFFF -> + (* Low surrogate reached without a consuming high half. *) + raise + (Lone_surrogate + "unsupported unpaired surrogate escape in string") + | Some _ -> i := !j + 5 + end + else i := !j + 1 + end + done + (* CROSS-LINEAGE CONTRACT: objects with duplicate member names (at any depth) must be rejected by all seven lineage libraries. yojson's Assoc is a plain pair list that preserves EVERY member — including @@ -76,6 +248,52 @@ let write_json_string buf s = s; Buffer.add_char buf '"' +(* ECMAScript-style shortest-roundtrip float formatting (RFC 8785 + §3.2.2.3 / ECMA-262 §7.1.12.1) for finite, non-integer-valued floats. + CROSS-LINEAGE CONTRACT: 0.1 must serialize as "0.1", not the %.17g + spelling "0.10000000000000001" — every lineage emits the SHORTEST + digit string that round-trips to the same IEEE 754 double. + + Shortest digits: the first precision p in 0..16 whose %.*e output + parses back to the exact value. Reassembly is always plain decimal — + the number-domain guard restricts fractions to |v| in [1e-6, 1e21), + so ECMA's exponent-notation branch is unreachable here. *) +let format_shortest_float f = + if f = 0. then "0" (* covers -0.0: canonical form is exactly "0" *) + else + let a = Float.abs f in + let rec shortest p = + if p > 16 then Printf.sprintf "%.17e" a + else + let s = Printf.sprintf "%.*e" p a in + if float_of_string s = a then s else shortest (p + 1) + in + let s = shortest 0 in + let epos = String.index s 'e' in + let mantissa = String.sub s 0 epos in + let exp = + int_of_string (String.sub s (epos + 1) (String.length s - epos - 1)) + in + let digits = String.concat "" (String.split_on_char '.' mantissa) in + (* Minimal p never ends in '0' (a shorter p would round-trip too), + but strip defensively so a stray zero can't shift the layout. *) + let digits = + let l = ref (String.length digits) in + while !l > 1 && digits.[!l - 1] = '0' do + decr l + done; + String.sub digits 0 !l + in + let k = String.length digits in + let n = exp + 1 in + let body = + if n >= k then digits ^ String.make (n - k) '0' + else if n > 0 then + String.sub digits 0 n ^ "." ^ String.sub digits n (k - n) + else "0." ^ String.make (-n) '0' ^ digits + in + if f < 0. then "-" ^ body else body + (** Recursively write a JSON value in canonical form. *) let rec canonicalize_value buf (v : Yojson.Safe.t) = match v with @@ -92,7 +310,7 @@ let rec canonicalize_value buf (v : Yojson.Safe.t) = lineage libraries emit 1.0 → "1", -3.0 → "-3", 1.5 → "1.5"; disagreement on this branch breaks SHA-256 digest parity. *) Buffer.add_string buf (Int64.to_string (Int64.of_float f)) - else Buffer.add_string buf (Printf.sprintf "%.17g" f) + else Buffer.add_string buf (format_shortest_float f) | `String s -> write_json_string buf s | `List items -> Buffer.add_char buf '['; diff --git a/ocaml/test/conformance_test.ml b/ocaml/test/conformance_test.ml index e39f9e5..e7aa660 100644 --- a/ocaml/test/conformance_test.ml +++ b/ocaml/test/conformance_test.ml @@ -164,6 +164,109 @@ let test_no_duplicates_mixed_allowed () = {|{"a":[1,2],"b":1}|} (Canonical_json.canonicalize_json v) +(* Number-domain enforcement is a LEXICAL pass over the raw text — + yojson parses 1e2 and 100 to the same value, so the exponent + spelling is only distinguishable before parsing. *) +let test_exponent_notation_rejected () = + Alcotest.check_raises "1e2 spelling rejected" + (Canonical_json.Number_rejected + "unsupported number (exponent notation): 1e2") + (fun () -> Canonical_json.reject_unsupported_numbers {|{"x":1e2}|}); + Alcotest.check_raises "uppercase 1E5 spelling rejected" + (Canonical_json.Number_rejected + "unsupported number (exponent notation): 1E5") + (fun () -> Canonical_json.reject_unsupported_numbers {|{"x":1E5}|}) + +let test_plain_100_allowed () = + Canonical_json.reject_unsupported_numbers {|{"x":100}|}; + (* 'e' inside a STRING literal is not a number token and must pass. *) + Canonical_json.reject_unsupported_numbers {|{"note":"1e2 and \"1E5\""}|}; + (* 'e' inside true/false keywords must not trip the scanner. *) + Canonical_json.reject_unsupported_numbers {|{"a":true,"b":false,"c":null}|} + +(* Integer bound compares digit strings, never floats: 9007199254740993 + rounds to 9007199254740992.0 and would slip past a float compare. *) +let test_integer_beyond_2_53_rejected () = + Alcotest.check_raises "2^53 + 1 rejected" + (Canonical_json.Number_rejected + "unsupported number (integer exceeds 2^53): 9007199254740993") + (fun () -> + Canonical_json.reject_unsupported_numbers {|{"x":9007199254740993}|}); + Alcotest.check_raises "-(2^53 + 1) rejected" + (Canonical_json.Number_rejected + "unsupported number (integer exceeds 2^53): -9007199254740993") + (fun () -> + Canonical_json.reject_unsupported_numbers {|{"x":-9007199254740993}|}); + (* 2^53 itself is the last representable integer and must pass. *) + Canonical_json.reject_unsupported_numbers + {|{"max_safe":9007199254740992,"neg":-9007199254740992}|} + +let test_fraction_out_of_range_rejected () = + Alcotest.check_raises "fraction below 1e-6 rejected" + (Canonical_json.Number_rejected + "unsupported number (fraction out of canonical range): 0.0000001") + (fun () -> + Canonical_json.reject_unsupported_numbers {|{"x":0.0000001}|}); + (* 1e-6 written plainly sits exactly on the boundary and must pass. *) + Canonical_json.reject_unsupported_numbers {|{"x":0.000001}|}; + Canonical_json.reject_unsupported_numbers {|{"x":0.1,"y":123.456,"z":0.5}|} + +(* Lone-surrogate enforcement is a LEXICAL pass over the raw text — + yojson maps a lone \udc00 escape to U+FFFD by parse time, which is + indistinguishable from a genuine U+FFFD in the input, so the + unpaired spelling is only visible before parsing. *) +let test_lone_low_surrogate_rejected () = + Alcotest.check_raises "lone low surrogate escape rejected" + (Canonical_json.Lone_surrogate + "unsupported unpaired surrogate escape in string") + (fun () -> Canonical_json.check_no_lone_surrogates {|{"s":"\udc00"}|}) + +let test_lone_high_surrogate_rejected () = + Alcotest.check_raises "lone high surrogate escape rejected" + (Canonical_json.Lone_surrogate + "unsupported unpaired surrogate escape in string") + (fun () -> Canonical_json.check_no_lone_surrogates {|{"s":"\ud800"}|}); + (* Case-insensitive hex: uppercase spelling is the same code unit. *) + Alcotest.check_raises "uppercase lone high surrogate escape rejected" + (Canonical_json.Lone_surrogate + "unsupported unpaired surrogate escape in string") + (fun () -> Canonical_json.check_no_lone_surrogates {|{"s":"\uD800"}|}) + +let test_surrogate_pair_allowed () = + (* A well-formed high+low escape pair (U+1F600, 😀) must pass: the + high half consumes its immediately-following low half. *) + Canonical_json.check_no_lone_surrogates {|{"s":"\ud83d\ude00"}|}; + (* Two pairs back-to-back: pair consumption must not skip past the + start of the second pair. *) + Canonical_json.check_no_lone_surrogates {|{"s":"\ud83d\ude00\ud83d\ude01"}|} + +let test_literal_backslash_udc00_allowed () = + (* \\udc00 is a LITERAL backslash followed by the text "udc00": the + escape starts only on the odd trailing backslash of a run, so an + even run pairs off into literal backslashes and must pass. *) + Canonical_json.check_no_lone_surrogates {|{"s":"\\udc00"}|} + +(* Shortest-roundtrip float formatting: 0.1 must canonicalize as "0.1", + not the %.17g spelling "0.10000000000000001" (RFC 8785 §3.2.2.3 / + ECMA-262 §7.1.12.1). *) +let test_shortest_roundtrip_floats () = + let check label expected v = + Alcotest.(check string) + label expected + (Canonical_json.canonicalize_json v) + in + check "0.1 shortest form" {|{"x":0.1}|} (`Assoc [ ("x", `Float 0.1) ]); + check "123.456 shortest form" {|{"x":123.456}|} + (`Assoc [ ("x", `Float 123.456) ]); + check "0.5 unchanged" {|{"x":0.5}|} (`Assoc [ ("x", `Float 0.5) ]); + check "negative fraction" {|{"x":-0.1}|} (`Assoc [ ("x", `Float (-0.1)) ]); + check "boundary 1e-6 as plain decimal" {|{"x":0.000001}|} + (`Assoc [ ("x", `Float 1e-6) ]); + (* Any zero — including negative zero — emits exactly "0". *) + check "negative zero is 0" {|{"x":0}|} (`Assoc [ ("x", `Float (-0.0)) ]); + (* Integer-valued floats keep the existing no-".0" behavior. *) + check "1.0 stays integer form" {|{"x":1}|} (`Assoc [ ("x", `Float 1.0) ]) + let () = Alcotest.run "BAION Canonical JSON Conformance" [ @@ -205,6 +308,33 @@ let () = Alcotest.test_case "unique keys with array value allowed" `Quick test_no_duplicates_mixed_allowed; ] ); + ( "number_domain", + [ + Alcotest.test_case "exponent notation rejected" `Quick + test_exponent_notation_rejected; + Alcotest.test_case "plain 100 and string/keyword 'e' allowed" `Quick + test_plain_100_allowed; + Alcotest.test_case "integer beyond 2^53 rejected" `Quick + test_integer_beyond_2_53_rejected; + Alcotest.test_case "fraction out of range rejected" `Quick + test_fraction_out_of_range_rejected; + ] ); + ( "lone_surrogate_rejection", + [ + Alcotest.test_case "lone low surrogate escape rejected" `Quick + test_lone_low_surrogate_rejected; + Alcotest.test_case "lone high surrogate escape rejected" `Quick + test_lone_high_surrogate_rejected; + Alcotest.test_case "surrogate pair escapes allowed" `Quick + test_surrogate_pair_allowed; + Alcotest.test_case "literal backslash + udc00 text allowed" `Quick + test_literal_backslash_udc00_allowed; + ] ); + ( "float_formatting", + [ + Alcotest.test_case "shortest-roundtrip float formatting" `Quick + test_shortest_roundtrip_floats; + ] ); ( "hash", [ Alcotest.test_case "SHA-256 reference vector" `Quick diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 22968e1..53be521 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baion_std" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "BAION canonical JSON + SHA-256 for Rust — public standalone library" license = "MIT" diff --git a/rust/src/bin/baion_canon_hash.rs b/rust/src/bin/baion_canon_hash.rs index 44101ad..042804a 100644 --- a/rust/src/bin/baion_canon_hash.rs +++ b/rust/src/bin/baion_canon_hash.rs @@ -5,12 +5,15 @@ // no whitespace, minimal escaping, integer-valued floats stripped of ".0"), // then prints the lowercase-hex SHA-256 of the canonical bytes + newline. // Exit 0 on success; exit 1 on parse error or on rejected input — any -// object key or string value containing U+0000, or any object with -// duplicate member names at any depth (message to stderr). +// object key or string value containing U+0000, any object with duplicate +// member names at any depth, or any number token outside the supported +// domain — exponent notation, integers past 2^53, fractions outside +// [1e-6, 1e21) (message to stderr). use baion_std::canonical_json::canonicalize_json; use baion_std::dup_check::check_duplicate_keys; use baion_std::hash::sha256_hex; +use baion_std::num_check::check_number_domain; use std::io::Read; use std::process::ExitCode; @@ -40,6 +43,15 @@ fn main() -> ExitCode { return ExitCode::from(1); } + // CROSS-LINEAGE CONTRACT (external review, 2026-07): the number domain + // is enforced LEXICALLY on the RAW input — after parse, 1e2 and 100 are + // the same f64, and out-of-range integer literals are already rounded, + // so only a raw-text scan can reject them. + if let Err(e) = check_number_domain(&input) { + eprintln!("baion_canon_hash: {}", e); + return ExitCode::from(1); + } + // CROSS-LINEAGE CONTRACT (external review, 2026-07): U+0000 anywhere in // an object key or string value is rejected — the library walk decides, // so the CLI and direct library callers agree on what is refused. diff --git a/rust/src/canonical_json.rs b/rust/src/canonical_json.rs index fbe1df4..281421d 100644 --- a/rust/src/canonical_json.rs +++ b/rust/src/canonical_json.rs @@ -70,9 +70,24 @@ pub fn canonicalize_value(v: &Value, out: &mut String) { if n.is_f64() { let f = n.as_f64().expect("is_f64 implies as_f64 is Some"); if f.is_finite() && f == f.trunc() && f.abs() <= 9007199254740992.0 { + // Casting also folds -0.0 to 0 ("-0" would diverge from + // the ES ToString reference, which emits "0"). out.push_str(&(f as i64).to_string()); return; } + // CROSS-LINEAGE CONTRACT: floats must serialize as PLAIN + // DECIMAL (ES ToString form), never exponent notation. + // serde_json's Number::to_string (Ryu writer) flips to + // exponent form below 1e-5 ("1e-6") and near the top of the + // domain ("1e20") — diverging from the C/Go/OCaml reference. + // Rust std Display for f64 is shortest-roundtrip and never + // uses exponent notation, which matches ES ToString exactly + // across the whole supported domain [1e-6, 1e21); the raw + // lexical gate (num_check) has already rejected anything + // outside it. Integer-valued floats past 2^53 (e.g. 1e20) + // land here too — Display gives the full digit string. + out.push_str(&format!("{}", f)); + return; } out.push_str(&n.to_string()); } @@ -195,6 +210,50 @@ mod tests { assert_eq!(result, r#"{"frac":1.5,"neg":-7,"vals":[1,2]}"#); } + #[test] + fn small_fractions_stay_plain_decimal() { + // Ryu's default writer flips to exponent form below 1e-5; the + // contract is ES ToString plain decimal down to the 1e-6 floor. + let j = json!({"a": 0.001_f64, "b": 0.0001_f64, "c": 0.00001_f64, "d": 0.000001_f64}); + let result = canonicalize_json(&j).unwrap(); + assert_eq!( + result, + r#"{"a":0.001,"b":0.0001,"c":0.00001,"d":0.000001}"# + ); + } + + #[test] + fn small_fraction_with_significand_plain_decimal() { + let j = json!({"x": 0.000123_f64}); + assert_eq!(canonicalize_json(&j).unwrap(), r#"{"x":0.000123}"#); + } + + #[test] + fn large_float_prints_full_digit_string() { + // 100000000000000000000.5 rounds to the double 1e20 — an + // integer-valued float past 2^53, so it skips the i64 cast and + // must still print as the 21-digit integer form, not "1e20". + let j: Value = serde_json::from_str(r#"{"x":100000000000000000000.5}"#).unwrap(); + assert_eq!( + canonicalize_json(&j).unwrap(), + r#"{"x":100000000000000000000}"# + ); + } + + #[test] + fn negative_zero_canonicalizes_to_zero() { + // ES ToString gives "0" for -0; "-0" would break byte parity. + let j: Value = serde_json::from_str(r#"{"x":-0.0}"#).unwrap(); + assert_eq!(canonicalize_json(&j).unwrap(), r#"{"x":0}"#); + } + + #[test] + fn ordinary_fractions_unchanged_by_plain_decimal_path() { + let j = json!({"a": 0.1_f64, "b": 123.456_f64, "c": 0.25_f64, "d": -0.375_f64}); + let result = canonicalize_json(&j).unwrap(); + assert_eq!(result, r#"{"a":0.1,"b":123.456,"c":0.25,"d":-0.375}"#); + } + #[test] fn empty_object() { let j = json!({}); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a54b056..099f4f7 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,3 +6,4 @@ pub mod canonical_json; pub mod dup_check; pub mod hash; +pub mod num_check; diff --git a/rust/src/num_check.rs b/rust/src/num_check.rs new file mode 100644 index 0000000..7f78903 --- /dev/null +++ b/rust/src/num_check.rs @@ -0,0 +1,270 @@ +// BAION canonical JSON for Rust — number-domain rejection pass. +// Validates raw JSON text before canonicalization: any number token outside +// the cross-lineage safe domain (exponent notation, integers beyond the +// double-exact range, fractions outside [1e-6, 1e21)) is rejected. +// Lineage: raw-input byte scanner, modeled on the D lineage's raw scanner +// (d/source/baionstd/canonical_json.d). +// +// CROSS-LINEAGE CONTRACT (external review, 2026-07): the number domain must +// be enforced LEXICALLY. serde_json hands the Visitor only f64/i64/u64 +// values, so `1e2` and `100` are indistinguishable after parse — a post-parse +// walk cannot see exponent notation, and huge integer literals have already +// been rounded. Only a scan of the raw text can distinguish them. The scan +// runs after the parse pass succeeds, so it may assume well-formed JSON +// (matched braces, valid strings, valid number grammar). + +use std::fmt; + +/// Rejection error: some number token in the input fell outside the +/// supported cross-lineage domain. `token` is the offending raw text. +#[derive(Debug, PartialEq, Eq)] +pub struct NumberDomainError { + pub token: String, + reason: &'static str, +} + +impl fmt::Display for NumberDomainError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "input contains unsupported number token {:?} ({}); rejected", + self.token, self.reason + ) + } +} + +impl std::error::Error for NumberDomainError {} + +// Largest integer n such that every integer in [-n, n] is exactly +// representable as an IEEE-754 double: 2^53. Compared as a DIGIT STRING so +// tokens too large for any native integer type are still judged correctly. +const MAX_SAFE_INTEGER_DIGITS: &str = "9007199254740992"; + +/// Scan raw JSON text and reject any number token outside the supported +/// domain. Returns Ok(()) when every number token is: +/// - free of exponent notation (`e`/`E`), +/// - if an integer (no `.`): within ±9007199254740992 (2^53, the +/// double-exact range), judged on the digit string, +/// - if a fraction (has `.`): zero, or with 1e-6 <= |v| < 1e21. +/// +/// Precondition: `input` is well-formed JSON (callers parse it first, as the +/// CLI does). The scanner skips string literals with escape handling, so +/// digits and `e` inside strings are never mistaken for number tokens. +pub fn check_number_domain(input: &str) -> Result<(), NumberDomainError> { + let bytes = input.as_bytes(); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'"' => i = skip_string(bytes, i), + b'-' | b'0'..=b'9' => { + let start = i; + // In well-formed JSON these bytes only ever appear inside a + // number token (strings were skipped above), so consuming the + // full number alphabet here cannot overrun the token. + while i < bytes.len() + && matches!(bytes[i], b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-') + { + i += 1; + } + check_token(&input[start..i])?; + } + _ => i += 1, + } + } + Ok(()) +} + +// Advance past a string literal, returning the index just after the closing +// quote. `\` consumes the next byte, so an escaped quote (`\"`) — or a +// backslash before it (`\\`) — never terminates the scan early. Escape +// payloads (`\uXXXX` hex, etc.) contain no `"` or `\`, so byte-stepping +// through them is safe. +fn skip_string(bytes: &[u8], open_quote: usize) -> usize { + let mut i = open_quote + 1; + while i < bytes.len() { + match bytes[i] { + b'\\' => i += 2, + b'"' => return i + 1, + _ => i += 1, + } + } + bytes.len() // unreachable on well-formed input +} + +fn check_token(token: &str) -> Result<(), NumberDomainError> { + let reject = |reason: &'static str| { + Err(NumberDomainError { + token: token.to_string(), + reason, + }) + }; + + // Exponent notation is rejected outright — canonical form across + // lineages never emits it in-domain, and accepting it would let `1e2` + // and `100` alias to the same canonical bytes from different sources. + if token.bytes().any(|b| b == b'e' || b == b'E') { + return reject("exponent notation is not supported"); + } + + if !token.contains('.') { + // Integer token: judge magnitude on the digit string, because a + // token like 99999999999999999999 exceeds every native integer type + // and must not be pushed through a lossy parse to be judged. + let digits = token + .strip_prefix('-') + .unwrap_or(token) + .trim_start_matches('0'); + let over = digits.len() > MAX_SAFE_INTEGER_DIGITS.len() + || (digits.len() == MAX_SAFE_INTEGER_DIGITS.len() + && digits > MAX_SAFE_INTEGER_DIGITS); + if over { + return reject("integer magnitude exceeds 9007199254740992"); + } + } else { + // Fraction token: the domain bounds are value-level (1e-6, 1e21), + // so an f64 parse is the right instrument here — unlike the integer + // case, in-domain fractions are exactly the doubles we canonicalize. + let v: f64 = token.parse().unwrap_or(f64::INFINITY); + if v != 0.0 && v.abs() < 1e-6 { + return reject("fraction magnitude below 1e-6"); + } + if v.abs() >= 1e21 { + return reject("fraction magnitude at or above 1e21"); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exponent_lowercase_rejected() { + let err = check_number_domain(r#"{"x":1e2}"#).unwrap_err(); + assert_eq!(err.token, "1e2"); + } + + #[test] + fn exponent_uppercase_rejected() { + let err = check_number_domain(r#"{"x":1E5}"#).unwrap_err(); + assert_eq!(err.token, "1E5"); + } + + #[test] + fn exponent_negative_rejected() { + let err = check_number_domain(r#"{"x":1e-7}"#).unwrap_err(); + assert_eq!(err.token, "1e-7"); + } + + #[test] + fn exponent_overflowing_rejected() { + // 1e400 overflows f64 to infinity — the lexical check fires first. + let err = check_number_domain(r#"{"x":1e400}"#).unwrap_err(); + assert_eq!(err.token, "1e400"); + } + + #[test] + fn plain_hundred_accepted() { + // The load-bearing distinction: 100 passes where 1e2 / 1E2 do not. + assert_eq!(check_number_domain(r#"{"x":100}"#), Ok(())); + assert!(check_number_domain(r#"{"x":1E2}"#).is_err()); + } + + #[test] + fn max_safe_integer_accepted_both_signs() { + assert_eq!( + check_number_domain(r#"{"a":9007199254740992,"b":-9007199254740992}"#), + Ok(()) + ); + } + + #[test] + fn integer_past_max_safe_rejected() { + let err = check_number_domain(r#"{"x":9007199254740993}"#).unwrap_err(); + assert_eq!(err.token, "9007199254740993"); + } + + #[test] + fn negative_integer_past_max_safe_rejected() { + let err = check_number_domain(r#"{"x":-9007199254740993}"#).unwrap_err(); + assert_eq!(err.token, "-9007199254740993"); + } + + #[test] + fn integer_beyond_native_width_rejected() { + // 21 digits — exceeds u64/i64; must be judged on the digit string. + let err = check_number_domain(r#"{"x":999999999999999999999}"#).unwrap_err(); + assert_eq!(err.token, "999999999999999999999"); + } + + #[test] + fn leading_zeros_do_not_inflate_magnitude() { + // Digit-string compare must strip leading zeros before length compare. + assert_eq!(check_number_domain(r#"[0.5]"#), Ok(())); + assert_eq!(check_number_domain(r#"{"x":0}"#), Ok(())); + } + + #[test] + fn tiny_fraction_rejected() { + let err = check_number_domain(r#"{"x":0.0000001}"#).unwrap_err(); + assert_eq!(err.token, "0.0000001"); + } + + #[test] + fn boundary_fraction_one_millionth_accepted() { + // |v| == 1e-6 exactly — inside the domain (strict < in the rule). + assert_eq!(check_number_domain(r#"{"x":0.000001}"#), Ok(())); + } + + #[test] + fn huge_fraction_rejected() { + let err = + check_number_domain(r#"{"x":10000000000000000000000.0}"#).unwrap_err(); + assert_eq!(err.token, "10000000000000000000000.0"); + } + + #[test] + fn zero_fractions_accepted() { + // 0.0 and -0.0 are exempt from the lower-magnitude bound. + assert_eq!(check_number_domain(r#"{"a":0.0,"b":-0.0}"#), Ok(())); + } + + #[test] + fn ordinary_fractions_accepted() { + assert_eq!(check_number_domain(r#"{"x":0.1,"y":1.0,"z":-2.5}"#), Ok(())); + } + + #[test] + fn numbers_inside_strings_ignored() { + // "1e400" is string CONTENT — the scanner must skip it. + assert_eq!(check_number_domain(r#"{"x":"1e400"}"#), Ok(())); + } + + #[test] + fn escaped_quote_does_not_leak_string_content() { + // The \" escape must not end the string early and expose 1e9. + assert_eq!(check_number_domain(r#"{"x":"a\"1e9\"b"}"#), Ok(())); + } + + #[test] + fn backslash_before_close_quote_handled() { + // "...\\" ends the string at the second quote; the 1e9 after the + // string (as a key's value) must still be caught. + let err = check_number_domain(r#"{"x":"c:\\","y":1e9}"#).unwrap_err(); + assert_eq!(err.token, "1e9"); + } + + #[test] + fn nested_structures_scanned() { + let err = check_number_domain(r#"{"a":[1,{"b":[2,3,1E2]}]}"#).unwrap_err(); + assert_eq!(err.token, "1E2"); + } + + #[test] + fn error_display_mentions_unsupported_number() { + let msg = check_number_domain(r#"{"x":1e2}"#).unwrap_err().to_string(); + assert!(msg.contains("unsupported")); + assert!(msg.contains("number")); + } +} diff --git a/verify_all_lineages.sh b/verify_all_lineages.sh index 7842696..1d9ad31 100755 --- a/verify_all_lineages.sh +++ b/verify_all_lineages.sh @@ -1,42 +1,24 @@ #!/usr/bin/env bash -# BAION STD — Cross-Lineage Byte-Identity Verifier -# Feeds identical JSON inputs to every lineage's baion_canon_hash CLI and -# asserts all SEVEN lineages emit the same SHA-256 of the same canonical -# bytes. A missing CLI is a FAILURE, not a skip: partial participation -# would let the strongest claim in this repo pass vacuously. +# BAION STD — Cross-Lineage Byte-Identity Verifier (corpus-driven) +# Feeds every conformance-corpus input to every lineage's baion_canon_hash CLI. +# Accept cases must hash to the corpus-PINNED SHA-256 in all SEVEN lineages; +# reject cases must exit nonzero in all seven. A missing CLI is a FAILURE, +# not a skip: partial participation would let the strongest claim in this +# repo pass vacuously. +# +# Vectors live in conformance/accept.jsonl + conformance/reject.jsonl (one +# JSON object per line; inputs are JSON-escaped strings so escape-sensitive +# bytes — NUL, BOM, lone surrogates — survive text editing losslessly). +# Regenerate/extend via conformance/gen_corpus.py, which refuses to pin any +# accept case the seven current CLIs disagree on. This script never invents +# an expected hash: pins come only from the corpus. set -u HERE="$(cd "$(dirname "$0")" && pwd)" LINEAGES=(c cpp rust go d haskell ocaml) EXPECTED=${#LINEAGES[@]} - -inputs=( - '{"b":1,"a":[1,2]}' - '{"z":1,"a":"é"}' - '{"nested":{"y":[true,false,null],"x":0.5},"empty":{},"arr":[]}' - '{"é":1,"e":2,"zß":"straße"}' - '{"escapes":"line\nbreak\ttab \"quoted\" back\\slash"}' - '{"max_safe":9007199254740992,"neg":-42,"empty":""}' - '{"deep":{"a":{"b":{"c":[1,{"d":[]}]}}}}' - '{"x":1.0}' - '1.0' - '{"x":"a\\u0000b"}' -) - -# Inputs every lineage must UNIFORMLY REJECT (nonzero exit). -# U+0000: outside the supported domain because one lineage cannot represent it losslessly — -# accepting it anywhere would allow silent canonicalization collisions. -# Duplicate object keys: RFC 8259 leaves duplicate-name behavior undefined and the seven -# ecosystems genuinely diverge (keep-first / keep-last / keep-both), so member names must -# be unique; duplicates compare on the DECODED name (third duplicate vector spells the -# same key as an escape). -reject_inputs=( - '{"x":"a\u0000b"}' - '{"a\u0000":1}' - '{"a":1,"a":2}' - '{"x":{"b":1,"b":2}}' - '{"a":1,"\u0061":2}' -) +ACCEPT="$HERE/conformance/accept.jsonl" +REJECT="$HERE/conformance/reject.jsonl" fail=0 missing=0 @@ -46,25 +28,49 @@ for L in "${LINEAGES[@]}"; do missing=1 fi done +for f in "$ACCEPT" "$REJECT"; do + if [ ! -f "$f" ]; then + echo "MISSING corpus file $f (run conformance/gen_corpus.py)" + missing=1 + fi +done if [ "$missing" -ne 0 ]; then - echo "FAIL: all $EXPECTED lineages must be built before verification" + echo "FAIL: all $EXPECTED lineages and both corpus files are required" exit 1 fi -for i in "${!inputs[@]}"; do - ref="" - echo "input $i: ${inputs[$i]}" +# Decode each corpus row to "namebase64(input-bytes)sha256" — base64 +# because the raw input bytes may contain NUL/BOM, which bash variables and +# command substitution cannot carry. +corpus_rows() { + python3 - "$1" <<'PY' +import base64, json, sys +for line in open(sys.argv[1], encoding="utf-8"): + line = line.strip() + if not line: + continue + row = json.loads(line) + payload = base64.b64encode(row["input"].encode("utf-8")).decode("ascii") + print(f'{row["name"]}\t{payload}\t{row.get("sha256", "")}') +PY +} + +# Accept pass: pinned-hash agreement across all seven lineages. +n_accept=0 +while IFS=$'\t' read -r name b64 pin; do + n_accept=$((n_accept + 1)) + echo "accept: $name" for L in "${LINEAGES[@]}"; do - h="$(printf '%s' "${inputs[$i]}" | "$HERE/$L/bin/baion_canon_hash")" || { echo " ERROR $L exited nonzero"; fail=1; continue; } - [ -z "$ref" ] && ref="$h" - if [ "$h" = "$ref" ]; then + h="$(printf '%s' "$b64" | base64 -d | "$HERE/$L/bin/baion_canon_hash")" \ + || { echo " ERROR $L exited nonzero"; fail=1; continue; } + if [ "$h" = "$pin" ]; then printf ' MATCH %-8s %s\n' "$L" "$h" else - printf ' DIFF %-8s %s\n' "$L" "$h" + printf ' DIFF %-8s %s (pinned %s)\n' "$L" "$h" "$pin" fail=1 fi done -done +done < <(corpus_rows "$ACCEPT") # Fixture file pass — the full cross-lineage conformance reference. ref="" @@ -75,21 +81,29 @@ for L in "${LINEAGES[@]}"; do if [ "$h" = "$ref" ]; then printf ' MATCH %-8s %s\n' "$L" "$h"; else printf ' DIFF %-8s %s\n' "$L" "$h"; fail=1; fi done -# Uniform-rejection pass: every lineage must refuse these with nonzero exit. -for i in "${!reject_inputs[@]}"; do - echo "reject $i: ${reject_inputs[$i]}" +# Reject pass: every lineage must refuse these with nonzero exit. Reasons are +# documented per-row in reject.jsonl. +n_reject=0 +while IFS=$'\t' read -r name b64 _; do + n_reject=$((n_reject + 1)) + echo "reject: $name" for L in "${LINEAGES[@]}"; do - if printf '%s' "${reject_inputs[$i]}" | "$HERE/$L/bin/baion_canon_hash" >/dev/null 2>&1; then + if printf '%s' "$b64" | base64 -d | "$HERE/$L/bin/baion_canon_hash" >/dev/null 2>&1; then printf ' BAD %-8s accepted input it must reject\n' "$L" fail=1 else printf ' REJECT %-7s ok\n' "$L" fi done -done +done < <(corpus_rows "$REJECT") + +if [ "$n_accept" -eq 0 ] || [ "$n_reject" -eq 0 ]; then + echo "FAIL: corpus is empty (accept=$n_accept reject=$n_reject) — refusing a vacuous pass" + fail=1 +fi if [ "$fail" -eq 0 ]; then - echo "PASS: $EXPECTED/$EXPECTED lineages produced identical output" + echo "PASS: $EXPECTED/$EXPECTED lineages agree on $n_accept accept + $n_reject reject vectors" else echo "FAIL: byte-identity failure" fi From 6b046b62f69f287e057d8ef06b23b9ed2afdb057 Mon Sep 17 00:00:00 2001 From: SaMullinsJr Date: Wed, 15 Jul 2026 13:29:47 -0400 Subject: [PATCH 2/2] =?UTF-8?q?harden:=20fuzz=20campaign=20round=20?= =?UTF-8?q?=E2=80=94=20uniform=20rejection=20of=20invalid=20UTF-8,=20raw?= =?UTF-8?q?=20controls,=20lax=20tokens;=20ES-262=20shortest-digits=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third conformance layer added: conformance/fuzz_agreement.py, a seeded randomized fuzzer (structured boundary-biased documents, byte mutations of valid documents, raw garbage) asserting seven-way agreement per input. Campaign seed 20260715, 8000 cases: initially 82 failures, now 0. Divergence classes closed (all lexical pre-parse checks; cores untouched): - invalid UTF-8 (stray continuation, truncated, overlong, encoded surrogate, >U+10FFFF): C/D/OCaml accepted verbatim, Go replaced with U+FFFD and hashed the replacement (silent cross-lineage collision) — now rejected in all 7 - raw control bytes in string literals (C, and pinned in Haskell/OCaml); non-whitespace control bytes between tokens (C skipped <=0x20; D skipped VT/FF via isWhite) — JSON whitespace is only TAB/LF/CR/space - number-token grammar: leading zeros (C, D), bare trailing dot (C), junk attached to number tokens '2-' (D) - malformed backslash-u escapes with <4 hex digits (C) - case-insensitive literals nuLl/falSe (D); unquoted keys, // and /**/ comments, NaN/Infinity/-Infinity (OCaml/yojson) - HASH SPLIT (worst class): Haskell emitted 17-digit exact value instead of ES-262 shortest digits for even-mantissa boundary doubles beyond 2^53 (GHC floatToDigits uses open-interval comparisons); shortening pass added, latent exact-integer fast path at 2^53-adjacent .0 tokens removed Corpus grown to 24 accept (pins big-float shortest-digits) + 31 reject rows; differential probe grown to 234 deterministic cases (raw-byte payload support for invalid-UTF-8 cases); README domain contract updated. Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + README.md | 10 +- c/include/baion/canonical_json.h | 38 ++++ c/src/canonical_json.c | 284 +++++++++++++++++++++++++ c/tests/test_canonical_json.c | 169 +++++++++++++++ c/tools/baion_canon_hash.c | 42 ++++ conformance/accept.jsonl | 1 + conformance/differential_probe.py | 58 ++++- conformance/fuzz_agreement.py | 212 ++++++++++++++++++ conformance/gen_corpus.py | 22 ++ conformance/reject.jsonl | 12 ++ d/source/baionstd/canonical_json.d | 249 ++++++++++++++++++++++ d/source/baionstd/types.d | 15 ++ d/tests/conformance_test.d | 167 +++++++++++++++ d/tools/canon_hash_main.d | 46 +++- go/canonical_json.go | 29 +++ go/canonical_json_test.go | 50 +++++ go/cmd/baion_canon_hash/main.go | 8 + haskell/app/Main.hs | 58 +++-- haskell/src/Baion/STD/CanonicalJson.hs | 126 +++++++++-- haskell/test/ConformanceTest.hs | 137 ++++++++++++ ocaml/cli/baion_canon_hash.ml | 24 +++ ocaml/lib/canonical_json.ml | 197 +++++++++++++++++ ocaml/test/conformance_test.ml | 162 ++++++++++++++ 24 files changed, 2072 insertions(+), 45 deletions(-) create mode 100644 conformance/fuzz_agreement.py diff --git a/.gitignore b/.gitignore index b6702a3..da3eb98 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ /*/.dub/ *.o *.a +conformance/__pycache__/ diff --git a/README.md b/README.md index 50134d8..0e36ab2 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Seven independent implementations of the same canonicalization contract — C, C **Prerequisites** (one toolchain per lineage): a C compiler + `make`; CMake ≥ 3.20 + a C++17 compiler; Rust (`cargo`); Go ≥ 1.22; D (`dmd` + `dub`); GHC ≥ 9.6 + `cabal` (aeson ≥ 2.2 is fetched by cabal); OCaml ≥ 5.x + `dune` with `yojson`, `digestif`, `alcotest` (via opam); `python3` for the conformance tooling. A successful run ends with a 7/7 PASS table from `build_all.sh` and `PASS: 7/7 lineages agree ...` from the verifier, exit 0. -`build_all.sh` builds each lineage with its native toolchain, runs its test suite, and places the CLI at `/bin/baion_canon_hash` — the layout the verifier requires. `verify_all_lineages.sh` then feeds every vector in the conformance corpus (`conformance/accept.jsonl` + `conformance/reject.jsonl`) to every CLI: accept vectors must hash to the corpus-pinned SHA-256 in all seven lineages, and reject vectors must be **uniformly refused** (see below). `conformance/differential_probe.py` additionally sweeps generated danger-zone cases (number bands, escape forms, document framing) and fails on any disagreement; both run in CI. All seven lineages must be present — a missing binary fails the run, and one byte of disagreement anywhere fails the run. Success prints `PASS: 7/7 lineages produced identical output`. +`build_all.sh` builds each lineage with its native toolchain, runs its test suite, and places the CLI at `/bin/baion_canon_hash` — the layout the verifier requires. `verify_all_lineages.sh` then feeds every vector in the conformance corpus (`conformance/accept.jsonl` + `conformance/reject.jsonl`) to every CLI: accept vectors must hash to the corpus-pinned SHA-256 in all seven lineages, and reject vectors must be **uniformly refused** (see below). `conformance/differential_probe.py` additionally sweeps generated danger-zone cases (number bands, escape forms, document framing) and fails on any disagreement; both run in CI. All seven lineages must be present — a missing binary fails the run, and one byte of disagreement anywhere fails the run. `conformance/fuzz_agreement.py` is the third layer: a seeded randomized fuzzer (structured boundary-biased documents, byte mutations, raw garbage) that asserts seven-way agreement on every generated input. ## Why this exists @@ -25,7 +25,7 @@ Cross-lineage byte-identity is enforced and tested for: - objects (member names must be **unique** — see below), arrays, strings (full UTF-8, including multi-byte and escaped control characters **except U+0000**), booleans, null - integers in the closed interval [−2⁵³, +2⁵³] (i.e. |n| ≤ 9007199254740992). Every integer in this interval is exactly representable in IEEE-754 binary64. Note this is one wider than JavaScript's `MAX_SAFE_INTEGER` (2⁵³ − 1): ±2⁵³ itself is admitted because it is exact and unambiguous *within this domain* — the neighboring value 2⁵³ + 1 (the first integer that would silently round to it) is rejected, so no two accepted integer tokens can collide -- floats in the **plain-decimal domain**: zero, or magnitude in [10⁻⁶, 10²¹), written without exponent notation; the canonical form is ECMAScript `ToString` (shortest round-trip digits, plain decimal) per RFC 8785 §3.2.2.3 — `1.0` → `1`, `-0.0` → `0`, `0.1` → `0.1`, in every lineage, enforced by the pinned corpus +- floats in the **plain-decimal domain**: zero, or magnitude in [10⁻⁶, 10²¹), written without exponent notation; the canonical form is ECMAScript `ToString` (shortest round-trip digits, plain decimal) per RFC 8785 §3.2.2.3 — including for values above 2⁵³ that enter via fraction tokens, where shortest-digits and the exact integer value differ (`65219416364867774.9377591` canonicalizes to `65219416364867780`, not `…776`) — `1.0` → `1`, `-0.0` → `0`, `0.1` → `0.1`, in every lineage, enforced by the pinned corpus **Uniformly rejected:** these input classes are refused with a nonzero exit by all seven CLIs, and the verifier asserts the rejection is uniform: @@ -34,8 +34,12 @@ Cross-lineage byte-identity is enforced and tested for: - *Numbers outside the plain-decimal domain*: exponent notation (`1e2` is rejected even though the value is in range — spell it `100`), integers beyond ±2⁵³, and fractions below 10⁻⁶ or at/above 10²¹. Seven number formatters genuinely disagree in exponent territory; the supported domain is exactly where byte-identity is provable, and inside it the output is normalized rather than excluded. - *Unpaired surrogate escapes* (a `\ud800`–`\udbff` escape not immediately followed by a low half, or a lone `\udc00`–`\udfff`): not Unicode scalar values, and ecosystems differ on replacement behavior. A literal backslash followed by surrogate text (`\\ud800`) is ordinary content. - *Anything other than exactly one JSON document*: a leading UTF-8 BOM, trailing non-whitespace, concatenated documents, trailing commas, or empty input. +- *Invalid UTF-8*: stray continuation bytes, truncated sequences, overlong encodings, encoded surrogates, and code points above U+10FFFF. One ecosystem silently replaces invalid bytes with U+FFFD before hashing — accepting invalid UTF-8 anywhere would allow silent canonicalization collisions. +- *Raw control characters* (U+0000–U+001F) inside string literals — RFC 8259 requires them escaped (`\t`, `\u001f`). Between tokens, only the four JSON whitespace bytes (tab, LF, CR, space) are accepted. +- *Malformed escapes and tokens*: an escape other than the eight RFC 8259 escapes or `\u` + exactly 4 hex digits; number tokens with leading zeros (`0635`), a bare trailing dot (`0.`), or attached junk (`2-`); literals not spelled exactly `null`/`true`/`false`. +- *JSON extensions* some parsers tolerate: comments (`//`, `/* */`), unquoted member names, and `NaN`/`Infinity`/`-Infinity` literals. -**Known exclusions:** non-finite numbers (NaN, ±Inf) are not valid JSON and are rejected or nulled per lineage test suites. The conformance corpus (`conformance/accept.jsonl`, pinned hashes; `conformance/reject.jsonl`, uniform rejections; regenerated by `conformance/gen_corpus.py`, which refuses to pin any case the seven CLIs disagree on) is the authoritative definition of the supported domain. If your data stays in the supported domain, the byte-identity guarantee holds. +**Known exclusions:** none beyond the rejection classes above. The conformance corpus (`conformance/accept.jsonl`, pinned hashes; `conformance/reject.jsonl`, uniform rejections; regenerated by `conformance/gen_corpus.py`, which refuses to pin any case the seven CLIs disagree on) is the authoritative definition of the supported domain. If your data stays in the supported domain, the byte-identity guarantee holds. ## Layout diff --git a/c/include/baion/canonical_json.h b/c/include/baion/canonical_json.h index efcb7ed..4fff8a7 100644 --- a/c/include/baion/canonical_json.h +++ b/c/include/baion/canonical_json.h @@ -29,6 +29,44 @@ int baion_reject_duplicate_keys(const cJSON* root); * being byte-distinct on the wire. */ int baion_reject_bom(const char* input, size_t len); +/* Pre-parse LEXICAL scan for raw control bytes: returns BAION_OK if the input + * carries no raw byte < 0x20 inside string literals and no raw byte < 0x20 + * other than TAB/LF/CR between tokens, else BAION_ERR_PARSE. Must run BEFORE + * cJSON parsing — cJSON accepts raw controls inside strings and skips any + * byte <= 0x20 between tokens as whitespace, both of which RFC 8259 forbids, + * so C would otherwise accept documents the sibling lineages reject. Escaped + * forms (the backslash-t and backslash-u001F spellings) stay accepted: they + * are escape TEXT, not raw bytes, so this byte-level check cannot + * false-positive on them. */ +int baion_reject_raw_controls(const char* input, size_t len); + +/* Pre-parse scan of raw input bytes: returns BAION_OK if the whole byte + * stream is well-formed UTF-8 per RFC 3629, else BAION_ERR_PARSE. Rejects + * continuation bytes without a lead, truncated sequences, overlong encodings + * (0xC0/0xC1 leads, 0xE0 0x80-0x9F, 0xF0 0x80-0x8F), encoded surrogates + * (0xED 0xA0-0xBF), and values above U+10FFFF (0xF4 0x90+, 0xF5-0xFF leads). + * Must run BEFORE cJSON parsing — cJSON copies string bytes through + * unexamined, so C would otherwise hash byte streams the sibling lineages + * reject at decode time. */ +int baion_reject_invalid_utf8(const char* input, size_t len); + +/* Pre-parse LEXICAL scan of escape shape inside string literals: returns + * BAION_OK if every backslash is followed by one of the eight single-char + * escapes (quote, backslash, slash, b, f, n, r, t) or by 'u' + exactly 4 hex + * digits, else BAION_ERR_PARSE. Must run BEFORE cJSON parsing — cJSON's hex + * decoding tolerates some short backslash-u forms. Surrogate PAIRING validity + * is out of scope here; this scan judges lexical shape only. */ +int baion_reject_malformed_escapes(const char* input, size_t len); + +/* Pre-parse LEXICAL scan of RFC 8259 number token shape: returns BAION_OK if + * every number token is optional '-', then '0' or [1-9] digits (no leading + * zeros), then optional '.' followed by at least one digit (no bare trailing + * dot), else BAION_ERR_PARSE. Exponent text is not judged here — that + * verdict belongs to baion_reject_number_domain. Must run BEFORE cJSON + * parsing — cJSON accepts leading zeros and bare trailing dots that the + * sibling lineages reject. */ +int baion_reject_number_grammar(const char* input, size_t len); + /* Pre-parse LEXICAL scan of raw JSON number tokens: returns BAION_OK if every * number token in the input is inside the plain-decimal domain, else * BAION_ERR_PARSE. Must run BEFORE cJSON parsing — cJSON collapses "100" and diff --git a/c/src/canonical_json.c b/c/src/canonical_json.c index efe6dcc..1ce97b6 100644 --- a/c/src/canonical_json.c +++ b/c/src/canonical_json.c @@ -393,6 +393,290 @@ int baion_reject_bom(const char* input, size_t len) return BAION_OK; } +/* CROSS-LINEAGE CONTRACT: raw control bytes are rejected LEXICALLY in every + * lineage per RFC 8259 — cJSON alone accepts raw controls inside strings AND + * skips any byte <= 0x20 between tokens as whitespace, so without this scan C + * would hash documents the sibling lineages reject. Escaped forms (the + * backslash-t and backslash-u001F spellings) are escape TEXT — bytes 0x5C + * 0x74 etc., never a raw byte < 0x20 — so a byte-level check cannot + * false-positive on them. */ +int baion_reject_raw_controls(const char* input, size_t len) +{ + int in_string = 0; + for (size_t i = 0; i < len; i++) + { + unsigned char c = (unsigned char)input[i]; + + if (in_string) + { + /* Inside a string literal EVERY control byte is illegal — RFC + * 8259 requires U+0000..U+001F to appear only in escaped form. */ + if (c < 0x20) + return BAION_ERR_PARSE; + if (c == '\\') + { + /* Escape neutralizes the next byte so an escaped quote + * cannot close the string — but that neutralized byte is + * still a raw byte and still must not be a control. */ + if (i + 1 < len) + { + i++; + if ((unsigned char)input[i] < 0x20) + return BAION_ERR_PARSE; + } + } + else if (c == '"') + in_string = 0; + } + else + { + if (c == '"') + in_string = 1; + /* Between tokens only TAB/LF/CR (and space, >= 0x20) are legal + * JSON whitespace — cJSON's skip wrongly treats 0x01..0x08, + * 0x0B, 0x0C, 0x0E..0x1F as skippable too. */ + else if (c < 0x20 && c != 0x09 && c != 0x0A && c != 0x0D) + return BAION_ERR_PARSE; + } + } + return BAION_OK; +} + +/* CROSS-LINEAGE CONTRACT: input byte streams must be well-formed UTF-8 per + * RFC 3629 in every lineage — cJSON copies string bytes through unexamined, + * so C alone would hash documents whose bytes the sibling lineages (C++, + * Rust, Haskell) reject at decode time. Runs over the WHOLE raw input, not + * just string interiors: any byte >= 0x80 outside a string is malformed JSON + * anyway, so whole-stream validation cannot false-positive on legal input. */ +int baion_reject_invalid_utf8(const char* input, size_t len) +{ + size_t i = 0; + while (i < len) + { + unsigned char b0 = (unsigned char)input[i]; + size_t cont; /* continuation bytes required after b0 */ + unsigned char lo = 0x80; /* legal range for the FIRST continuation */ + unsigned char hi = 0xBF; /* byte — tightened per lead to exclude */ + /* overlong forms, surrogates, > U+10FFFF */ + if (b0 <= 0x7F) + { + i++; + continue; + } + else if (b0 >= 0xC2 && b0 <= 0xDF) + cont = 1; + else if (b0 == 0xE0) + { + /* E0 80-9F would re-encode U+0000..U+07FF overlong */ + cont = 2; + lo = 0xA0; + } + else if ((b0 >= 0xE1 && b0 <= 0xEC) || b0 == 0xEE || b0 == 0xEF) + cont = 2; + else if (b0 == 0xED) + { + /* ED A0-BF encodes U+D800..U+DFFF — surrogates are not scalar + * values and RFC 3629 forbids their encoded form outright. */ + cont = 2; + hi = 0x9F; + } + else if (b0 == 0xF0) + { + /* F0 80-8F would re-encode U+0000..U+FFFF overlong */ + cont = 3; + lo = 0x90; + } + else if (b0 >= 0xF1 && b0 <= 0xF3) + cont = 3; + else if (b0 == 0xF4) + { + /* F4 90+ encodes values above U+10FFFF */ + cont = 3; + hi = 0x8F; + } + else + { + /* 0x80-0xBF: continuation byte without a lead. + * 0xC0/0xC1: leads that can only produce overlong encodings. + * 0xF5-0xFF: leads for values above U+10FFFF (or not UTF-8). */ + return BAION_ERR_PARSE; + } + + if (i + cont >= len) + return BAION_ERR_PARSE; /* truncated sequence at end of input */ + unsigned char b1 = (unsigned char)input[i + 1]; + if (b1 < lo || b1 > hi) + return BAION_ERR_PARSE; + for (size_t k = 2; k <= cont; k++) + { + unsigned char bk = (unsigned char)input[i + k]; + if (bk < 0x80 || bk > 0xBF) + return BAION_ERR_PARSE; + } + i += cont + 1; + } + return BAION_OK; +} + +/* CROSS-LINEAGE CONTRACT: escape SHAPE is enforced lexically in every lineage + * per RFC 8259 §7 — cJSON's parse_hex4 tolerates fewer than 4 hex digits in + * some malformed inputs (fuzzer round 5: the 5-char "backslash-u-0-0-e-s" and + * short "backslash-u-d-8-3" forms parsed), so without this scan C would hash + * documents the sibling lineages reject. Pairing validity of surrogate + * escapes is a separate concern handled after decode; this scan checks only + * lexical shape: one of the eight single-char escapes, or 'u' + exactly 4 hex + * digits. */ +int baion_reject_malformed_escapes(const char* input, size_t len) +{ + int in_string = 0; + for (size_t i = 0; i < len; i++) + { + char c = input[i]; + if (!in_string) + { + if (c == '"') + in_string = 1; + continue; + } + if (c == '"') + { + in_string = 0; + continue; + } + if (c != '\\') + continue; + + if (i + 1 >= len) + return BAION_ERR_PARSE; /* backslash at end of input */ + char e = input[++i]; + switch (e) + { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + break; + case 'u': + { + if (i + 4 >= len) + return BAION_ERR_PARSE; /* truncated backslash-u escape */ + for (size_t k = 1; k <= 4; k++) + { + char h = input[i + k]; + if (!((h >= '0' && h <= '9') || (h >= 'a' && h <= 'f') + || (h >= 'A' && h <= 'F'))) + return BAION_ERR_PARSE; + } + i += 4; + break; + } + default: + return BAION_ERR_PARSE; /* not one of the eight escapes or 'u' */ + } + } + return BAION_OK; +} + +/* CROSS-LINEAGE CONTRACT: RFC 8259 §6 number SHAPE is enforced lexically in + * every lineage — cJSON accepts leading zeros ("0635", "-004") and bare + * trailing dots ("0."), which the sibling lineages reject at parse time. + * Shape checked here: optional '-', then '0' or [1-9] digits (no leading + * zeros), then optional '.' followed by AT LEAST one digit. Exponent text is + * NOT judged here — baion_reject_number_domain already rejects every e/E + * token, and this scan must not disturb that verdict. */ +int baion_reject_number_grammar(const char* input, size_t len) +{ + size_t i = 0; + while (i < len) + { + char c = input[i]; + + if (c == '"') + { + /* Skip string literals — same escape rule as the sibling scans: + * a backslash neutralizes the next char so an escaped quote + * cannot close the string. */ + i++; + while (i < len && input[i] != '"') + { + if (input[i] == '\\' && i + 1 < len) + i++; + i++; + } + i++; + continue; + } + + if (c == '-' || (c >= '0' && c <= '9')) + { + /* Consume the same token alphabet as baion_reject_number_domain + * so both scans agree on token extent. */ + size_t start = i; + int has_exp = 0; + while (i < len) + { + char t = input[i]; + if (t >= '0' && t <= '9') + { + /* digit: always part of the token */ + } + else if (t == '.') + { + /* dot: part of the token (validity judged below) */ + } + else if (t == 'e' || t == 'E') + has_exp = 1; + else if ((t == '+' || t == '-') && (i == start || has_exp)) + { + /* sign: leading minus or exponent sign only */ + } + else + break; + i++; + } + + const char* p = input + start; + const char* q = input + i; + if (*p == '-') + p++; + if (p >= q || *p < '0' || *p > '9') + return BAION_ERR_PARSE; /* '-' with no integer part */ + if (*p == '0') + { + p++; + if (p < q && *p >= '0' && *p <= '9') + return BAION_ERR_PARSE; /* leading zero: 0635, -004, 01. */ + } + else + { + while (p < q && *p >= '0' && *p <= '9') + p++; + } + if (p < q && *p == '.') + { + p++; + if (p >= q || *p < '0' || *p > '9') + return BAION_ERR_PARSE; /* bare trailing dot: 0. / 01. */ + while (p < q && *p >= '0' && *p <= '9') + p++; + } + /* Any residue must be exponent text — that verdict belongs to + * baion_reject_number_domain (which rejects all e/E tokens). + * Non-exponent residue (e.g. a second dot) is shape-invalid. */ + if (p < q && *p != 'e' && *p != 'E') + return BAION_ERR_PARSE; + continue; + } + + i++; + } + return BAION_OK; +} + /* CROSS-LINEAGE CONTRACT: the plain-decimal number domain is enforced * LEXICALLY over the raw token in every lineage — "100" is in-domain while * "1e2" is not, even though both parse to the same double. A post-parse diff --git a/c/tests/test_canonical_json.c b/c/tests/test_canonical_json.c index bd0882f..dafa7cc 100644 --- a/c/tests/test_canonical_json.c +++ b/c/tests/test_canonical_json.c @@ -270,6 +270,171 @@ static void test_bom_rejection(void) ASSERT_INT_EQ(baion_reject_bom(interior, strlen(interior)), BAION_OK); } +static void test_raw_control_rejection(void) +{ + /* Raw control byte INSIDE a string literal: RFC 8259 requires it to be + * escaped — cJSON alone would accept it. The "\t" / "\x1e" C escapes + * below put the raw byte in the runtime payload, never in this source. */ + const char* raw_tab = "{\"s\":\"a\tb\"}"; + ASSERT_INT_EQ(baion_reject_raw_controls(raw_tab, strlen(raw_tab)), BAION_ERR_PARSE); + const char* raw_rs = "{\"s\":\"a\x1e" + "b\"}"; + ASSERT_INT_EQ(baion_reject_raw_controls(raw_rs, strlen(raw_rs)), BAION_ERR_PARSE); + const char* raw_after_backslash = "{\"s\":\"\\\x01\"}"; + ASSERT_INT_EQ(baion_reject_raw_controls(raw_after_backslash, strlen(raw_after_backslash)), + BAION_ERR_PARSE); + + /* Non-whitespace control byte BETWEEN tokens: cJSON's skip treats any + * byte <= 0x20 as whitespace, but only TAB/LF/CR/space are legal. */ + const char* ctrl_between = "{\"a\":1,\x02\"b\":2}"; + ASSERT_INT_EQ(baion_reject_raw_controls(ctrl_between, strlen(ctrl_between)), BAION_ERR_PARSE); + + /* Legal JSON whitespace between tokens: allowed */ + const char* tab_ws = "{\"a\":\t1}"; + ASSERT_INT_EQ(baion_reject_raw_controls(tab_ws, strlen(tab_ws)), BAION_OK); + const char* crlf_ws = "\n{\"a\":1}\r\n"; + ASSERT_INT_EQ(baion_reject_raw_controls(crlf_ws, strlen(crlf_ws)), BAION_OK); + + /* Escaped forms are escape TEXT (bytes 0x5C 0x74 / 0x5C 'u' ...), not + * raw control bytes — must stay accepted, including behind an escaped + * quote that must not close the string. */ + const char* esc_tab = "{\"s\":\"a\\tb\"}"; + ASSERT_INT_EQ(baion_reject_raw_controls(esc_tab, strlen(esc_tab)), BAION_OK); + const char* esc_u001f = "{\"s\":\"\\u001f\"}"; + ASSERT_INT_EQ(baion_reject_raw_controls(esc_u001f, strlen(esc_u001f)), BAION_OK); + const char* esc_quote_then_raw_ok = "{\"a\\\"b\":\"\\n\"}"; + ASSERT_INT_EQ(baion_reject_raw_controls(esc_quote_then_raw_ok, strlen(esc_quote_then_raw_ok)), + BAION_OK); + + /* TAB is only whitespace OUTSIDE strings — a raw TAB after an escaped + * quote is still inside the string literal and must be rejected. */ + const char* raw_tab_after_esc_quote = "{\"a\\\"\t\":1}"; + ASSERT_INT_EQ( + baion_reject_raw_controls(raw_tab_after_esc_quote, strlen(raw_tab_after_esc_quote)), + BAION_ERR_PARSE); +} + +static void test_invalid_utf8_rejection(void) +{ + /* All payloads spell high bytes with C \xNN escapes — never raw bytes in + * this source file. */ + + /* Lead byte followed by a non-continuation byte */ + const char* bad_lead = "{\"\xe0" + "a\":[]}"; + ASSERT_INT_EQ(baion_reject_invalid_utf8(bad_lead, strlen(bad_lead)), BAION_ERR_PARSE); + + /* Continuation byte without a lead */ + const char* stray_cont = "\"a\x85" + "b\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(stray_cont, strlen(stray_cont)), BAION_ERR_PARSE); + + /* Overlong encodings: C0/C1 leads, E0 80-9F, F0 80-8F */ + const char* overlong2 = "\"\xc0\xaf\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(overlong2, strlen(overlong2)), BAION_ERR_PARSE); + const char* overlong3 = "\"\xe0\x9f\xbf\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(overlong3, strlen(overlong3)), BAION_ERR_PARSE); + const char* overlong4 = "\"\xf0\x8f\xbf\xbf\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(overlong4, strlen(overlong4)), BAION_ERR_PARSE); + + /* Encoded surrogate U+D800 (ED A0 80) */ + const char* enc_surrogate = "\"\xed\xa0\x80\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(enc_surrogate, strlen(enc_surrogate)), + BAION_ERR_PARSE); + + /* Values above U+10FFFF: F4 90+ and F5-FF leads */ + const char* above_max = "\"\xf4\x90\x80\x80\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(above_max, strlen(above_max)), BAION_ERR_PARSE); + const char* f5_lead = "\"\xf5\x80\x80\x80\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(f5_lead, strlen(f5_lead)), BAION_ERR_PARSE); + + /* Truncated sequence at end of input */ + const char* truncated = "\"\xc3"; + ASSERT_INT_EQ(baion_reject_invalid_utf8(truncated, strlen(truncated)), BAION_ERR_PARSE); + + /* Well-formed multi-byte content stays accepted: e-acute (C3 A9), + * boundary 3-byte forms (E0 A0 80, ED 9F BF), 4-byte emoji (F0 9F 98 80), + * top of the range (F4 8F BF BF). */ + const char* e_acute = "{\"a\":\"\xc3\xa9\"}"; + ASSERT_INT_EQ(baion_reject_invalid_utf8(e_acute, strlen(e_acute)), BAION_OK); + const char* boundary3 = "\"\xe0\xa0\x80 \xed\x9f\xbf\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(boundary3, strlen(boundary3)), BAION_OK); + const char* emoji = "{\"x\":\"\xf0\x9f\x98\x80\"}"; + ASSERT_INT_EQ(baion_reject_invalid_utf8(emoji, strlen(emoji)), BAION_OK); + const char* top = "\"\xf4\x8f\xbf\xbf\""; + ASSERT_INT_EQ(baion_reject_invalid_utf8(top, strlen(top)), BAION_OK); +} + +static void test_malformed_escape_rejection(void) +{ + /* Short backslash-u forms the fuzzer caught cJSON accepting */ + const char* short_u = "\"\\u00es\""; + ASSERT_INT_EQ(baion_reject_malformed_escapes(short_u, strlen(short_u)), BAION_ERR_PARSE); + const char* short_u2 = "\"\\ud83\\ude00\""; + ASSERT_INT_EQ(baion_reject_malformed_escapes(short_u2, strlen(short_u2)), BAION_ERR_PARSE); + const char* trunc_u = "\"\\u12"; + ASSERT_INT_EQ(baion_reject_malformed_escapes(trunc_u, strlen(trunc_u)), BAION_ERR_PARSE); + + /* Backslash before a char outside the eight escapes / 'u' */ + const char* bad_esc = "\"a\\qb\""; + ASSERT_INT_EQ(baion_reject_malformed_escapes(bad_esc, strlen(bad_esc)), BAION_ERR_PARSE); + + /* All eight legal single-char escapes plus a full backslash-u escape */ + const char* legal = "\"\\\" \\\\ \\/ \\b \\f \\n \\r \\t \\u001f\""; + ASSERT_INT_EQ(baion_reject_malformed_escapes(legal, strlen(legal)), BAION_OK); + + /* Escaped quote must not close the string: the backslash-u after it is + * still inside the literal and still judged. */ + const char* esc_quote_then_bad = "{\"a\\\"\\u12g\":1}"; + ASSERT_INT_EQ(baion_reject_malformed_escapes(esc_quote_then_bad, strlen(esc_quote_then_bad)), + BAION_ERR_PARSE); + + /* Outside strings a backslash is not judged here (cJSON rejects it as a + * syntax error) — no false positive on structural text. */ + const char* outside = "{\"a\":1}"; + ASSERT_INT_EQ(baion_reject_malformed_escapes(outside, strlen(outside)), BAION_OK); +} + +static void test_number_grammar_rejection(void) +{ + /* Leading zeros */ + const char* lead_zero = "0635"; + ASSERT_INT_EQ(baion_reject_number_grammar(lead_zero, strlen(lead_zero)), BAION_ERR_PARSE); + const char* lead_zero_neg = "-004"; + ASSERT_INT_EQ(baion_reject_number_grammar(lead_zero_neg, strlen(lead_zero_neg)), + BAION_ERR_PARSE); + const char* zero_one = "{\"k\":01}"; + ASSERT_INT_EQ(baion_reject_number_grammar(zero_one, strlen(zero_one)), BAION_ERR_PARSE); + const char* zero_one_dot = "01.5"; + ASSERT_INT_EQ(baion_reject_number_grammar(zero_one_dot, strlen(zero_one_dot)), + BAION_ERR_PARSE); + + /* Bare trailing dot */ + const char* trail_dot = "{\"k\":0.}"; + ASSERT_INT_EQ(baion_reject_number_grammar(trail_dot, strlen(trail_dot)), BAION_ERR_PARSE); + const char* trail_dot2 = "[1.]"; + ASSERT_INT_EQ(baion_reject_number_grammar(trail_dot2, strlen(trail_dot2)), BAION_ERR_PARSE); + + /* Bare minus / dot-first */ + const char* bare_minus = "[-]"; + ASSERT_INT_EQ(baion_reject_number_grammar(bare_minus, strlen(bare_minus)), BAION_ERR_PARSE); + const char* minus_dot = "-.5"; + ASSERT_INT_EQ(baion_reject_number_grammar(minus_dot, strlen(minus_dot)), BAION_ERR_PARSE); + + /* Legal shapes stay accepted, including bare 0, -0.0 and fractions */ + const char* legal = "{\"a\":0,\"b\":-0.0,\"c\":0.5,\"d\":123,\"e\":-42,\"f\":0.000001}"; + ASSERT_INT_EQ(baion_reject_number_grammar(legal, strlen(legal)), BAION_OK); + + /* Exponent text is not judged here — the number-domain scan owns that + * verdict (and rejects it). */ + const char* exp_tok = "1e2"; + ASSERT_INT_EQ(baion_reject_number_grammar(exp_tok, strlen(exp_tok)), BAION_OK); + + /* Digits inside string literals are not number tokens */ + const char* in_string = "{\"s\":\"0635 and 0. here\"}"; + ASSERT_INT_EQ(baion_reject_number_grammar(in_string, strlen(in_string)), BAION_OK); +} + int main(void) { printf("test_canonical_json:\n"); @@ -284,6 +449,10 @@ int main(void) RUN_TEST(test_float_shortest_roundtrip); RUN_TEST(test_number_domain_rejection); RUN_TEST(test_bom_rejection); + RUN_TEST(test_raw_control_rejection); + RUN_TEST(test_invalid_utf8_rejection); + RUN_TEST(test_malformed_escape_rejection); + RUN_TEST(test_number_grammar_rejection); TEST_SUMMARY(); return _test_failed; } diff --git a/c/tools/baion_canon_hash.c b/c/tools/baion_canon_hash.c index d48af1e..5aa567a 100644 --- a/c/tools/baion_canon_hash.c +++ b/c/tools/baion_canon_hash.c @@ -64,6 +64,17 @@ int main(void) return 1; } + /* Pre-parse UTF-8 well-formedness rejection (library-level scan): cJSON + copies string bytes through unexamined, so invalid byte sequences the + sibling lineages reject at decode time would hash here. Reviewer + contract: reject with exit 1. */ + if (baion_reject_invalid_utf8(input, len) != BAION_OK) + { + fprintf(stderr, "baion_canon_hash: invalid UTF-8 byte sequence in input\n"); + free(input); + return 1; + } + /* Pre-parse U+0000 rejection (library-level scan): cJSON decodes the u0000 escape into a NUL byte that truncates the C string, so distinct documents would collapse onto one canonical form. Reviewer contract: @@ -75,6 +86,37 @@ int main(void) return 1; } + /* Pre-parse raw-control rejection (library-level LEXICAL scan): cJSON + accepts raw control bytes inside strings and skips non-whitespace + control bytes between tokens, both forbidden by RFC 8259 — the sibling + lineages reject them. Reviewer contract: reject with exit 1. */ + if (baion_reject_raw_controls(input, len) != BAION_OK) + { + fprintf(stderr, "baion_canon_hash: raw control byte in input is unsupported\n"); + free(input); + return 1; + } + + /* Pre-parse escape-shape rejection (library-level LEXICAL scan): cJSON + tolerates some short backslash-u forms that the sibling lineages + reject. Reviewer contract: reject with exit 1. */ + if (baion_reject_malformed_escapes(input, len) != BAION_OK) + { + fprintf(stderr, "baion_canon_hash: malformed string escape in input is invalid\n"); + free(input); + return 1; + } + + /* Pre-parse number-shape rejection (library-level LEXICAL scan): cJSON + accepts leading zeros and bare trailing dots that RFC 8259 forbids and + the sibling lineages reject. Reviewer contract: reject with exit 1. */ + if (baion_reject_number_grammar(input, len) != BAION_OK) + { + fprintf(stderr, "baion_canon_hash: invalid number token shape in input\n"); + free(input); + return 1; + } + /* Pre-parse number-domain rejection (library-level LEXICAL scan): cJSON collapses "100" and "1e2" onto the same double, so only the raw token spelling can enforce the plain-decimal domain (no exponent notation, diff --git a/conformance/accept.jsonl b/conformance/accept.jsonl index 1588905..74308c3 100644 --- a/conformance/accept.jsonl +++ b/conformance/accept.jsonl @@ -21,3 +21,4 @@ {"name": "deep_nest_60", "input": "{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":{\"a\":1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}", "sha256": "8f86b55fdc801b4c73916389d1f83f057412a6e333132529f2fb44bc192956d0"} {"name": "same_key_sibling_objects", "input": "[{\"k\":1},{\"k\":2}]", "sha256": "98fcf287e1991c1602a189793606501715f8ae194db5dcaaf6515ed29937c20d"} {"name": "micro_boundary", "input": "{\"x\":0.000001}", "sha256": "2d6412ab0155bb89d63b73dbf83334b26b39d03e8c832cc253c6a6caceba9733"} +{"name": "big_float_shortest_digits", "input": "{\"x\":65219416364867774.9377591}", "sha256": "96df26a780f11e7a70c701e23ff9f79ab115118d77c70945885996a1014a2d84"} diff --git a/conformance/differential_probe.py b/conformance/differential_probe.py index 912f9dc..ea9342d 100644 --- a/conformance/differential_probe.py +++ b/conformance/differential_probe.py @@ -75,6 +75,62 @@ def build_cases(): cases.append(("str_control_escapes", '{"s":"' + BS + 'u0001' + BS + 'u001f"}')) cases.append(("str_long", '{"s":"' + "xy" * 512 + '"}')) + # Raw control bytes: rejected inside strings (RFC 8259 §7 requires the + # escape), and outside strings only 0x09/0x0A/0x0D/0x20 are whitespace. + for b in [0x01, 0x02, 0x09, 0x0A, 0x0D, 0x1E, 0x1F]: + cases.append((f"str_raw_ctrl_{b:02x}", '{"s":"a' + chr(b) + 'b"}')) # reject + cases.append((f"key_raw_ctrl_{b:02x}", '{"a' + chr(b) + '":1}')) # reject + for b in [0x01, 0x02, 0x0B, 0x0C, 0x1F]: + cases.append((f"ws_raw_ctrl_{b:02x}", '{"a":1,' + chr(b) + '"b":2}')) # reject + cases.append(("ws_tab_between_tokens", '{"a":' + chr(0x09) + '1}')) # legal ws + cases.append(("ws_crlf_framing", chr(0x0D) + chr(0x0A) + '{"a":1}' + chr(0x0D) + chr(0x0A))) + + # Raw invalid UTF-8 (bytes payloads — cannot be expressed as str). One + # lineage replaced bad bytes with U+FFFD and hashed the result: a silent + # cross-lineage collision, the worst divergence class. Uniform reject. + Q, OPEN, CLOSE = b'"', b'{"', b'":[]}' + cases.append(("utf8_stray_lead", OPEN + bytes([0xE0]) + b"a" + CLOSE)) # reject + cases.append(("utf8_stray_continuation", Q + b"a" + bytes([0x85]) + b"b" + Q)) # reject + cases.append(("utf8_overlong_slash", Q + bytes([0xC0, 0xAF]) + Q)) # reject + cases.append(("utf8_encoded_surrogate", Q + bytes([0xED, 0xA0, 0x80]) + Q)) # reject + cases.append(("utf8_truncated_2byte", Q + b"a" + bytes([0xC3]) + Q)) # reject + cases.append(("utf8_beyond_10ffff", Q + bytes([0xF4, 0x90, 0x80, 0x80]) + Q)) # reject + cases.append(("utf8_f5_lead", Q + bytes([0xF5, 0x80, 0x80, 0x80]) + Q)) # reject + cases.append(("utf8_valid_2byte", '{"x":"é"}')) # accept + cases.append(("utf8_valid_4byte", '{"x":"\U0001F600"}')) # accept + + # Number-token grammar edges (RFC 8259: int = 0 / [1-9]digits; frac needs a digit). + for v in ["0635", "-004", "01", "007", "03000000000000004"]: + cases.append((f"num_leadzero_{v}", v)) # reject + for v in ["0.", "01.", ".5", "-.5", "+1", "-", "2-", "1-", "77-7957", "0635.5"]: + cases.append((f"num_malformed_{v}", v)) # reject + cases.append(("num_zero_frac_ok", '{"x":0.25}')) # accept + + # Literal spellings must be exact. + for v in ["nuLl", "falSe", "tRue", "TRUE", "nul", "truee"]: + cases.append((f"lit_{v}", v)) # reject + + # Non-JSON extensions some parsers allow: comments, non-finite literals. + cases.append(("line_comment", '{"a":1} // note')) # reject + cases.append(("line_comment_inside", '{"a": // note\n1}')) # reject + cases.append(("block_comment", '{"a":/* note */1}')) # reject + for v in ["NaN", "Infinity", "-Infinity", "True", "None"]: + cases.append((f"nonfinite_{v}", '{"x":' + v + "}")) # reject + cases.append(("nan_in_string_ok", '{"x":"NaN // fine"}')) # accept + + # Structure lexing: unquoted keys, malformed escapes. + cases.append(("unquoted_key", "{tz:true}")) # reject + cases.append(("short_u_escape", '"' + BS + 'u00es"')) # reject + cases.append(("short_u_surrogate", '"' + BS + 'ud83' + BS + 'ude00"')) # reject + cases.append(("unknown_escape_q", '"a' + BS + 'qb"')) # reject + cases.append(("escape_at_eof", '"a' + BS)) # reject + + # Integer-valued doubles beyond 2^53 entering via fraction tokens: + # canonical form is ES-262 SHORTEST digits, not the exact integer value. + cases.append(("big_float_shortest", "65219416364867774.9377591")) # accept, one hash + cases.append(("big_float_shortest_2", '{"x":9007199254740993.5}')) + cases.append(("big_float_shortest_3", "123456789012345678.9")) + # Structure: duplicates, depth, document framing. cases.append(("dup_toplevel", '{"a":1,"a":2}')) # reject cases.append(("dup_nested", '{"x":{"b":1,"b":2}}')) # reject @@ -103,7 +159,7 @@ def main() -> int: cases = build_cases() divergent = [] for name, text in cases: - payload = text.encode("utf-8") + payload = text.encode("utf-8") if isinstance(text, str) else text results = {L: run_cli(L, payload) for L in LINEAGES} accepts = {L for L, (rc, _) in results.items() if rc == 0} if accepts and accepts != set(LINEAGES): diff --git a/conformance/fuzz_agreement.py b/conformance/fuzz_agreement.py new file mode 100644 index 0000000..cc31c41 --- /dev/null +++ b/conformance/fuzz_agreement.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# BAION STD agreement fuzzer — Observer (randomized adversarial sweep) +# Spec: repo README "Supported JSON domain"; harden-v0.2.0 conformance suite. +# +# WHY this exists: the corpus pins known vectors and differential_probe.py +# sweeps hand-enumerated danger zones; this fuzzer generates cases nobody +# thought of. Three attack classes: (1) structured documents biased toward +# the domain boundaries (2^53, 1e-6/1e21, surrogate range, escape forms), +# (2) byte-level mutations of valid documents, (3) raw garbage. For every +# input the seven CLIs must agree — all accept with one hash, or all reject +# — and none may die on a signal. The RNG is seeded from argv so any failure +# reproduces exactly: rerun with the same seed and case index. +# +# Usage: fuzz_agreement.py [seed] [n_structured] [n_mutation] [n_garbage] +import random +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +LINEAGES = ["c", "cpp", "rust", "go", "d", "haskell", "ocaml"] +BS = chr(0x5C) + +BOUNDARY_INTS = [0, 1, -1, 2**53, -(2**53), 2**53 - 1, 2**53 + 1, + 2**63, 2**64, 10**20, 10**21, 999, -42] +BOUNDARY_FRACTION_TEXTS = [ + "0.000001", "0.0000001", "0.0000015", "0.00001", "0.1", "0.5", + "123.456", "-0.375", "-0.0", "0.0", "1.0", "3.141592653589793", + "100000000000000000000.5", "999999999999999999999.9", + "0.30000000000000004", "1.7976931348623157", +] +NASTY_STRING_PARTS = [ + "plain", "sp ace", "é", "ß", "中文", "\U0001F600", + "�", "￿", "line1\nline2", "tab\there", 'quo"te', "back" + BS + "slash", + BS + "n", BS + "t", BS + '"', BS + BS, BS + "/", BS + "u0041", + BS + "u00e9", BS + "ud83d" + BS + "ude00", # escape TEXT, inserted raw + BS + "u0000", BS + "ud800", BS + "udc00", # must force uniform rejection + BS + BS + "u0000", BS + BS + "ud800", # literal-backslash forms: fine + "e", "E", "1e5", "-0.0", "null", "true", "9007199254740993", +] + + +def rand_key(rng, depth): + kind = rng.randrange(6) + if kind == 0: + return "k%d" % rng.randrange(1000) + if kind == 1: + return rng.choice(["a", "b", "aa", "ab", "z", "é", "e"]) + if kind == 2: + return "" + if kind == 3: + return rng.choice(NASTY_STRING_PARTS) + if kind == 4: + return "k" + str(depth) + return "".join(rng.choice("abz09_") for _ in range(rng.randrange(1, 9))) + + +def rand_number_text(rng): + kind = rng.randrange(5) + if kind == 0: + return str(rng.choice(BOUNDARY_INTS)) + if kind == 1: + return rng.choice(BOUNDARY_FRACTION_TEXTS) + if kind == 2: + return str(rng.randrange(-10**6, 10**6)) + if kind == 3: # random plain fraction, digit-built (no float formatting) + whole = str(rng.randrange(0, 10**rng.randrange(1, 18))) + frac = "".join(rng.choice("0123456789") for _ in range(rng.randrange(1, 12))) + return ("-" if rng.random() < 0.4 else "") + whole + "." + frac + # exponent spellings — outside the domain, must uniformly reject + return "%d%s%d" % (rng.randrange(1, 999), rng.choice("eE"), + rng.randrange(-400, 400)) + + +def rand_string_text(rng): + n = rng.randrange(0, 5) + return "".join(rng.choice(NASTY_STRING_PARTS) for _ in range(n)) + + +def rand_value_text(rng, depth): + """Build JSON source TEXT directly (not via json.dumps) so escape + sequences land in the document as typed, exactly like hostile input.""" + if depth > 5 or rng.random() < 0.3: + leaf = rng.randrange(5) + if leaf == 0: + return rand_number_text(rng) + if leaf == 1: + return '"' + rand_string_text(rng) + '"' + return rng.choice(["true", "false", "null"]) + if rng.random() < 0.5: + n = rng.randrange(0, 5) + items = ",".join(rand_value_text(rng, depth + 1) for _ in range(n)) + return "[" + items + "]" + n = rng.randrange(0, 5) + seen, members = set(), [] + for _ in range(n): + k = rand_key(rng, depth) + dup = rng.random() < 0.03 # occasionally inject a duplicate on purpose + if k in seen and not dup: + continue + seen.add(k) + members.append('"' + k + '":' + rand_value_text(rng, depth + 1)) + return "{" + ",".join(members) + "}" + + +def mutate(rng, data: bytes) -> bytes: + buf = bytearray(data) + for _ in range(rng.randrange(1, 4)): + op = rng.randrange(4) + if not buf: + break + i = rng.randrange(len(buf)) + if op == 0: + buf[i] = rng.randrange(256) + elif op == 1: + del buf[i] + elif op == 2: + buf.insert(i, rng.randrange(256)) + else: + j = rng.randrange(len(buf)) + buf[i], buf[j] = buf[j], buf[i] + return bytes(buf) + + +def rand_garbage(rng) -> bytes: + kind = rng.randrange(4) + n = rng.randrange(0, 64) + if kind == 0: + return bytes(rng.randrange(256) for _ in range(n)) + if kind == 1: + return bytes(rng.randrange(0x20, 0x7F) for _ in range(n)) + if kind == 2: # JSON-ish token soup + toks = ['{', '}', '[', ']', ':', ',', '"a"', '1', 'true', 'null', + ' ', BS, '"', '1e', '-', '.', '0'] + return "".join(rng.choice(toks) for _ in range(n)).encode() + return ("" * rng.randrange(1, 3) + '{"a":1}').encode() + + +def run_cli(lineage: str, payload: bytes): + cli = ROOT / lineage / "bin" / "baion_canon_hash" + p = subprocess.run([str(cli)], input=payload, capture_output=True, timeout=20) + return p.returncode, p.stdout.decode(errors="replace").strip() + + +def check(payload: bytes): + """Return None if the seven agree and none crashed, else a report dict.""" + results = {L: run_cli(L, payload) for L in LINEAGES} + crashed = {L: rc for L, (rc, _) in results.items() if rc < 0} + if crashed: + return {"kind": "crash", "results": results, "crashed": crashed} + accepts = {L for L, (rc, _) in results.items() if rc == 0} + if accepts and accepts != set(LINEAGES): + return {"kind": "accept/reject split", "results": results} + hashes = {h for rc, h in results.values() if rc == 0} + if len(hashes) > 1: + return {"kind": "hash split", "results": results} + return None + + +def main() -> int: + seed = int(sys.argv[1]) if len(sys.argv) > 1 else 20260715 + n_structured = int(sys.argv[2]) if len(sys.argv) > 2 else 4000 + n_mutation = int(sys.argv[3]) if len(sys.argv) > 3 else 3000 + n_garbage = int(sys.argv[4]) if len(sys.argv) > 4 else 1000 + rng = random.Random(seed) + failures = [] + accepted = rejected = 0 + valid_pool = [] + + def record(tag, idx, payload, report): + failures.append((tag, idx, payload, report)) + print(f"FAIL {tag}#{idx} ({report['kind']}): " + f"{payload[:80]!r}{'...' if len(payload) > 80 else ''}") + for L, (rc, h) in report["results"].items(): + print(f" {L:8s} {'rc=' + str(rc) if rc != 0 else h}") + + phases = [ + ("structured", n_structured, + lambda: rand_value_text(rng, 0).encode("utf-8")), + ("mutation", n_mutation, + lambda: mutate(rng, rng.choice(valid_pool)) if valid_pool + else rand_garbage(rng)), + ("garbage", n_garbage, lambda: rand_garbage(rng)), + ] + for tag, count, gen in phases: + for i in range(count): + payload = gen() + report = check(payload) + if report is None: + rc0, _ = run_cli(LINEAGES[0], payload) + if rc0 == 0: + accepted += 1 + if tag == "structured" and len(valid_pool) < 500: + valid_pool.append(payload) + else: + rejected += 1 + else: + record(tag, i, payload, report) + if (i + 1) % 1000 == 0: + print(f" {tag}: {i + 1}/{count} " + f"(uniform-accept={accepted} uniform-reject={rejected} " + f"fail={len(failures)})", flush=True) + + total = n_structured + n_mutation + n_garbage + print(f"seed={seed} cases={total} uniform-accept={accepted} " + f"uniform-reject={rejected} FAILURES={len(failures)}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/gen_corpus.py b/conformance/gen_corpus.py index 1066a4a..38e83d5 100644 --- a/conformance/gen_corpus.py +++ b/conformance/gen_corpus.py @@ -50,6 +50,9 @@ ("deep_nest_60", ('{"a":' * 60) + "1" + ("}" * 60)), ("same_key_sibling_objects", '[{"k":1},{"k":2}]'), ("micro_boundary", '{"x":0.000001}'), + # Pins ES-262 shortest-digits for an integer-valued double beyond 2^53: + # exact value is ...776 but the canonical spelling is the 16-digit ...780. + ("big_float_shortest_digits", '{"x":65219416364867774.9377591}'), ] # Uniform-rejection contract. reason is documentation for humans + remediation maps. @@ -73,7 +76,26 @@ ("two_documents", '{"a":1}{"b":2}', "input must be exactly one JSON document"), ("trailing_comma", '{"a":1,}', "not valid RFC 8259 JSON"), ("empty_input", "", "input must be exactly one JSON document"), + ("raw_tab_in_string", '{"s":"a' + chr(0x09) + 'b"}', + "RFC 8259 requires control characters in strings to be escaped; three lineages accepted raw TAB"), + ("raw_ctrl_1e_in_string", '"tr' + chr(0x1E) + 'R"', + "raw control byte inside string literal; escape it"), + ("raw_lf_in_string", '{"s":"a' + chr(0x0A) + 'b"}', + "raw newline inside string literal; spell it as an escape"), + ("ctrl_between_tokens", '{"a":1,' + chr(0x02) + '"b":2}', + "only TAB/LF/CR/space are JSON whitespace; one lineage skipped any byte <= 0x20"), + ("leading_zero_int", "0635", "RFC 8259 int part is 0 or [1-9]digits; two lineages accepted"), + ("neg_leading_zero_int", "-004", "RFC 8259 int part is 0 or [1-9]digits"), + ("trailing_dot_number", '{"k":0.}', "fraction part requires at least one digit"), + ("number_trailing_junk", "2-", "trailing bytes after a bare number token"), + ("case_insensitive_literal", "nuLl", "literals must be spelled exactly null/true/false"), + ("unquoted_key", "{tz:true}", "object member names must be quoted strings"), + ("short_u_escape", '"' + BS + 'u00es"', "backslash-u requires exactly 4 hex digits"), + ("unknown_escape", '"a' + BS + 'qb"', "only the eight RFC 8259 escapes plus u are legal"), ] +# NOTE: invalid-UTF-8 rejection (the fuzzer's largest class) cannot be pinned here — +# the corpus input field is a JSON string, which cannot carry invalid byte sequences. +# That class is covered by differential_probe.py (raw-bytes cases) and the fuzzer. def run_cli(lineage: str, payload: bytes): diff --git a/conformance/reject.jsonl b/conformance/reject.jsonl index 4579510..a14c445 100644 --- a/conformance/reject.jsonl +++ b/conformance/reject.jsonl @@ -17,3 +17,15 @@ {"name": "two_documents", "input": "{\"a\":1}{\"b\":2}", "reason": "input must be exactly one JSON document"} {"name": "trailing_comma", "input": "{\"a\":1,}", "reason": "not valid RFC 8259 JSON"} {"name": "empty_input", "input": "", "reason": "input must be exactly one JSON document"} +{"name": "raw_tab_in_string", "input": "{\"s\":\"a\tb\"}", "reason": "RFC 8259 requires control characters in strings to be escaped; three lineages accepted raw TAB"} +{"name": "raw_ctrl_1e_in_string", "input": "\"tr\u001eR\"", "reason": "raw control byte inside string literal; escape it"} +{"name": "raw_lf_in_string", "input": "{\"s\":\"a\nb\"}", "reason": "raw newline inside string literal; spell it as an escape"} +{"name": "ctrl_between_tokens", "input": "{\"a\":1,\u0002\"b\":2}", "reason": "only TAB/LF/CR/space are JSON whitespace; one lineage skipped any byte <= 0x20"} +{"name": "leading_zero_int", "input": "0635", "reason": "RFC 8259 int part is 0 or [1-9]digits; two lineages accepted"} +{"name": "neg_leading_zero_int", "input": "-004", "reason": "RFC 8259 int part is 0 or [1-9]digits"} +{"name": "trailing_dot_number", "input": "{\"k\":0.}", "reason": "fraction part requires at least one digit"} +{"name": "number_trailing_junk", "input": "2-", "reason": "trailing bytes after a bare number token"} +{"name": "case_insensitive_literal", "input": "nuLl", "reason": "literals must be spelled exactly null/true/false"} +{"name": "unquoted_key", "input": "{tz:true}", "reason": "object member names must be quoted strings"} +{"name": "short_u_escape", "input": "\"\\u00es\"", "reason": "backslash-u requires exactly 4 hex digits"} +{"name": "unknown_escape", "input": "\"a\\qb\"", "reason": "only the eight RFC 8259 escapes plus u are legal"} diff --git a/d/source/baionstd/canonical_json.d b/d/source/baionstd/canonical_json.d index 0fcd6a5..72a78d3 100644 --- a/d/source/baionstd/canonical_json.d +++ b/d/source/baionstd/canonical_json.d @@ -519,6 +519,255 @@ private bool isJSONWhitespace(char c) @safe pure nothrow return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } +// ── Inter-token control-byte enforcement scan ──────────────── +// +// CROSS-LINEAGE CONTRACT: outside string literals only the four RFC +// 8259 §2 whitespace bytes (0x09 TAB, 0x0A LF, 0x0D CR, 0x20 space) +// may separate tokens. All 7 lineages reject any other byte below +// 0x20 between tokens — the CLI exits 1 naming an unsupported +// control character. +// +// WHY a raw-input scan: std.json's parseJSON delegates inter-token +// skipping to isWhite (measured, dmd 2.112.0), which also accepts +// 0x0B vertical tab and 0x0C form feed — so `{"a":1,\x0B"b":2}` +// parses cleanly and the defect is invisible post-parse. +// +// Raw control bytes INSIDE string literals are NOT this scan's job: +// parseJSON already rejects them ("Illegal control character", +// measured dmd 2.112.0) before any raw scan runs, so string literals +// are consumed whole here. +// +// Precondition: as with the other raw scanners, parseJSON has already +// accepted the input, so every string literal is terminated. + +/// Scan raw JSON text for control bytes between tokens that are not +/// JSON whitespace. Returns true if any such byte exists. +bool hasUnsupportedControl(const(char)[] raw) +{ + size_t i = 0; + while (i < raw.length) + { + char c = raw[i]; + if (c == '"') + { + // Control bytes inside string literals are parseJSON's + // jurisdiction (see header): consume the whole literal. + decodeJSONString(raw, i); + continue; + } + if (c < 0x20 && !isJSONWhitespace(c)) + return true; + i++; + } + return false; +} + +// ── Raw UTF-8 enforcement scan ─────────────────────────────── +// +// CROSS-LINEAGE CONTRACT: the raw input bytes must be well-formed +// RFC 3629 UTF-8. All 7 lineages reject stray continuation bytes, +// truncated sequences, overlong encodings (0xC0/0xC1, 0xE0 0x80-0x9F, +// 0xF0 0x80-0x8F), encoded surrogates (0xED 0xA0-0xBF) and codepoints +// above U+10FFFF (0xF4 0x90+, 0xF5-0xFF) — the CLI exits 1 naming +// invalid UTF-8. +// +// WHY a raw-input scan: std.json's parseJSON (measured, dmd 2.112.0) +// passes non-ASCII bytes through untouched — `{"\xE0a":[]}` and a bare +// `"a\x85b"` parse cleanly and the invalid bytes land in the canonical +// output, diverging from the C++/Rust/Haskell lineages which reject. +// +// UNLIKE the other raw scanners this one has NO parseJSON precondition: +// it runs FIRST, on the raw bytes, before any parse — the other +// scanners (and decodeJSONString) may then assume valid UTF-8. + +/// Scan raw input bytes for RFC 3629 well-formedness violations. +/// Returns true if any invalid UTF-8 sequence exists. +bool hasInvalidUTF8(const(ubyte)[] raw) @safe pure nothrow +{ + size_t i = 0; + while (i < raw.length) + { + immutable ubyte b = raw[i]; + if (b < 0x80) + { + i++; + continue; + } + + // Sequence length and the RFC 3629 §4 tightened range of the + // SECOND byte — this is where overlong forms, surrogates and + // > U+10FFFF are excluded, not just the 0x80-0xBF shape. + size_t len; + ubyte lo2 = 0x80, hi2 = 0xBF; + if (b >= 0xC2 && b <= 0xDF) + len = 2; + else if (b == 0xE0) + { + len = 3; + lo2 = 0xA0; // 0x80-0x9F would be an overlong 2-byte value + } + else if (b >= 0xE1 && b <= 0xEC) + len = 3; + else if (b == 0xED) + { + len = 3; + hi2 = 0x9F; // 0xA0-0xBF would encode a UTF-16 surrogate + } + else if (b >= 0xEE && b <= 0xEF) + len = 3; + else if (b == 0xF0) + { + len = 4; + lo2 = 0x90; // 0x80-0x8F would be an overlong 3-byte value + } + else if (b >= 0xF1 && b <= 0xF3) + len = 4; + else if (b == 0xF4) + { + len = 4; + hi2 = 0x8F; // 0x90-0xBF would encode above U+10FFFF + } + else + { + // 0x80-0xBF stray continuation, 0xC0/0xC1 overlong leads, + // 0xF5-0xFF out-of-range leads. + return true; + } + + if (i + len > raw.length) + return true; // truncated sequence + if (raw[i + 1] < lo2 || raw[i + 1] > hi2) + return true; + foreach (k; 2 .. len) + { + if (raw[i + k] < 0x80 || raw[i + k] > 0xBF) + return true; + } + i += len; + } + return false; +} + +// ── Strict token-lexeme enforcement scan ───────────────────── +// +// CROSS-LINEAGE CONTRACT: number tokens must match the RFC 8259 §6 +// grammar exactly (int part is `0` or [1-9] digits — no leading zeros, +// no junk bytes attached to the token), and literal tokens must be +// exactly `null`, `true`, `false` — never case variants. All 7 +// lineages reject `0635`, `-004`, `2-`, `77-7957`, `nuLl`, `falSe`, +// `tRue` — the CLI exits 1 naming the invalid token. +// +// WHY a raw-input scan: std.json's parseJSON (measured, dmd 2.112.0) +// accepts leading-zero integers, hashes `2-` / `77-7957` as the +// leading number while silently dropping the tail, and matches the +// null/true/false keywords CASE-INSENSITIVELY — post-parse none of +// this lexical evidence survives. scanSingleDocument cannot catch the +// attached-junk shape either: its greedy number sweep (matching +// parseJSON's own lexer) swallows `-` bytes into the token, so the +// junk never surfaces as trailing data — malformed number lexemes are +// THIS scanner's jurisdiction. +// +// Precondition: hasInvalidUTF8 has already accepted the bytes; +// parseJSON has already accepted the input (lexeme laxities above +// notwithstanding), so string literals are terminated. + +/// Scan raw JSON text for number tokens violating the RFC 8259 grammar +/// and for literal tokens that are not exactly null/true/false. +/// Returns the violation, or a null Nullable when every token is clean. +Nullable!StdError scanStrictTokens(const(char)[] raw) +{ + Nullable!StdError err; + + size_t i = 0; + while (i < raw.length) + { + immutable char c = raw[i]; + if (c == '"') + { + // Digits/letters inside string literals are not tokens: + // consume the whole literal (decoded text is discarded). + decodeJSONString(raw, i); + continue; + } + if (c == '-' || (c >= '0' && c <= '9')) + { + // Greedy sweep over the number alphabet — the same lexer view + // parseJSON takes, so junk it silently attached (`2-`) lands + // inside the token and fails the grammar check below. + immutable size_t start = i; + while (i < raw.length && (raw[i] == '-' || raw[i] == '+' + || raw[i] == '.' || raw[i] == 'e' || raw[i] == 'E' + || (raw[i] >= '0' && raw[i] <= '9'))) + i++; + if (!numberTokenGrammarValid(raw[start .. i])) + { + err = StdError.invalidNumber; + return err; + } + continue; + } + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) + { + // Literal token: sweep the alphabetic run and demand an exact + // keyword. 'e'/'E' inside numbers never reach here — the + // number sweep above owns them. + immutable size_t start = i; + while (i < raw.length && ((raw[i] >= 'a' && raw[i] <= 'z') + || (raw[i] >= 'A' && raw[i] <= 'Z'))) + i++; + const tok = raw[start .. i]; + if (tok != "null" && tok != "true" && tok != "false") + { + err = StdError.invalidLiteral; + return err; + } + continue; + } + i++; + } + return err; +} + +/// Check one raw number token against the RFC 8259 §6 grammar: +/// -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)? +/// (Exponent forms are grammar-VALID here; hasUnsupportedNumber owns +/// the domain rejection of exponents and out-of-range magnitudes.) +private bool numberTokenGrammarValid(const(char)[] tok) @safe pure nothrow +{ + size_t i = 0; + if (i < tok.length && tok[i] == '-') + i++; + // int part: `0` alone, or a nonzero digit then any digits — a `0` + // followed by another digit is the leading-zero shape RFC 8259 forbids. + if (i >= tok.length || tok[i] < '0' || tok[i] > '9') + return false; + if (tok[i] == '0') + i++; + else + while (i < tok.length && tok[i] >= '0' && tok[i] <= '9') + i++; + if (i < tok.length && tok[i] == '.') + { + i++; + if (i >= tok.length || tok[i] < '0' || tok[i] > '9') + return false; + while (i < tok.length && tok[i] >= '0' && tok[i] <= '9') + i++; + } + if (i < tok.length && (tok[i] == 'e' || tok[i] == 'E')) + { + i++; + if (i < tok.length && (tok[i] == '+' || tok[i] == '-')) + i++; + if (i >= tok.length || tok[i] < '0' || tok[i] > '9') + return false; + while (i < tok.length && tok[i] >= '0' && tok[i] <= '9') + i++; + } + // Any leftover byte is junk parseJSON attached to the token (`2-`). + return i == tok.length; +} + /// Decode one JSON string literal starting at the opening quote. /// Advances `i` past the closing quote; returns the decoded text. /// WHY decode at all: key comparison must match the parser's view, diff --git a/d/source/baionstd/types.d b/d/source/baionstd/types.d index 5c17bd3..54dcafc 100644 --- a/d/source/baionstd/types.d +++ b/d/source/baionstd/types.d @@ -12,6 +12,10 @@ enum StdError emptyInput, /// Empty or whitespace-only input (no JSON document at all) trailingData, /// Content after the first complete JSON document trailingComma, /// Comma immediately before '}' or ']' + unsupportedControl, /// Raw control byte between tokens that is not JSON whitespace + invalidUTF8, /// Raw input bytes are not well-formed RFC 3629 UTF-8 + invalidNumber, /// Number token violates RFC 8259 grammar (leading zero / attached junk) + invalidLiteral, /// Literal token is not exactly `null`, `true`, or `false` } /// Human-readable error messages. @@ -33,5 +37,16 @@ string errorMessage(StdError e) @safe pure nothrow return "trailing data after the JSON document is rejected (exactly one JSON document is required)"; case StdError.trailingComma: return "trailing comma before '}' or ']' is rejected"; + case StdError.unsupportedControl: + return "unsupported control character outside a string literal is rejected" + ~ " (JSON whitespace is only tab, LF, CR, space)"; + case StdError.invalidUTF8: + return "invalid UTF-8 in raw input is rejected (RFC 3629: stray continuation," + ~ " truncated or overlong sequence, encoded surrogate, or above U+10FFFF)"; + case StdError.invalidNumber: + return "invalid number token is rejected (RFC 8259: leading zero in the" + ~ " integer part, or junk bytes attached to the number)"; + case StdError.invalidLiteral: + return "invalid literal is rejected (must be exactly null, true, or false)"; } } diff --git a/d/tests/conformance_test.d b/d/tests/conformance_test.d index a386f25..094a6bc 100644 --- a/d/tests/conformance_test.d +++ b/d/tests/conformance_test.d @@ -384,3 +384,170 @@ unittest assert(scanSingleDocument(`{"a":1,"b":[1,2]}`).isNull, "single-document FAILED: legitimate commas wrongly flagged"); } + +// ── Inter-token control-byte scan tests ── +// Only 0x09/0x0A/0x0D/0x20 may separate tokens (RFC 8259 §2); std.json's +// isWhite-based skip also accepts 0x0B/0x0C, so the raw scan must catch +// them. Raw control bytes are built with \xNN escapes — never literal +// control bytes in this source file. +unittest +{ + // Vertical tab / form feed between tokens — the differential-probe bug + assert(hasUnsupportedControl("{\"a\":1,\x0B\"b\":2}"), + "control-scan FAILED: vertical tab between tokens not flagged"); + assert(hasUnsupportedControl("{\"a\":1,\x0C\"b\":2}"), + "control-scan FAILED: form feed between tokens not flagged"); + + // Other control bytes: leading, trailing, and interior positions + assert(hasUnsupportedControl("\x0B{\"a\":1}"), + "control-scan FAILED: leading vertical tab not flagged"); + assert(hasUnsupportedControl("{\"a\":1}\x0C"), + "control-scan FAILED: trailing form feed not flagged"); + assert(hasUnsupportedControl("{\"a\":\x021}"), + "control-scan FAILED: STX between tokens not flagged"); + assert(hasUnsupportedControl("[1,\x1F2]"), + "control-scan FAILED: 0x1F between tokens not flagged"); + assert(hasUnsupportedControl("\x00"), + "control-scan FAILED: bare NUL not flagged"); + + // Legal RFC 8259 whitespace must pass untouched + assert(!hasUnsupportedControl("\t\r\n {\"a\":\t1} \r\n"), + "control-scan FAILED: legal whitespace wrongly flagged"); + + // Escaped controls inside strings are TEXT, not raw bytes — pass + assert(!hasUnsupportedControl("{\"s\":\"abc\\nd\"}"), + "control-scan FAILED: escaped in-string controls wrongly flagged"); + + // Whitespace-looking bytes inside string literals belong to the + // string (parseJSON polices raw in-string controls) — the scan must + // consume literals whole, not flag their contents + assert(!hasUnsupportedControl("{\"s\":\"a b\"}"), + "control-scan FAILED: space inside string wrongly flagged"); +} + +// ── Raw UTF-8 scan tests ── +// RFC 3629 well-formedness on the raw bytes: parseJSON passes invalid +// UTF-8 through untouched, so the raw scan must reject it first. All +// non-ASCII payload bytes are built with \xNN escapes — never raw +// bytes in this source file. +unittest +{ + alias b = (string s) => cast(const(ubyte)[]) s; + + // The agreement-fuzzer counterexamples + assert(hasInvalidUTF8(b("{\"\xe0a\":[]}")), + "utf8-scan FAILED: truncated 3-byte lead in key not flagged"); + assert(hasInvalidUTF8(b("\"a\x85b\"")), + "utf8-scan FAILED: stray continuation byte not flagged"); + + // Truncated sequences (lead byte with too few continuations) + assert(hasInvalidUTF8(b("\"\xc3\"")), + "utf8-scan FAILED: truncated 2-byte sequence not flagged"); + assert(hasInvalidUTF8(b("\"\xe2\x82\"")), + "utf8-scan FAILED: truncated 3-byte sequence not flagged"); + assert(hasInvalidUTF8(b("\xf0\x9f\x98")), + "utf8-scan FAILED: truncated 4-byte sequence at EOF not flagged"); + + // Overlong encodings + assert(hasInvalidUTF8(b("\"\xc0\xaf\"")), + "utf8-scan FAILED: overlong 0xC0 not flagged"); + assert(hasInvalidUTF8(b("\"\xc1\x81\"")), + "utf8-scan FAILED: overlong 0xC1 not flagged"); + assert(hasInvalidUTF8(b("\"\xe0\x80\xa0\"")), + "utf8-scan FAILED: overlong 0xE0 0x80 not flagged"); + assert(hasInvalidUTF8(b("\"\xe0\x9f\xbf\"")), + "utf8-scan FAILED: overlong 0xE0 0x9F not flagged"); + assert(hasInvalidUTF8(b("\"\xf0\x8f\xbf\xbf\"")), + "utf8-scan FAILED: overlong 0xF0 0x8F not flagged"); + + // Encoded UTF-16 surrogates + assert(hasInvalidUTF8(b("\"\xed\xa0\x80\"")), + "utf8-scan FAILED: encoded high surrogate not flagged"); + assert(hasInvalidUTF8(b("\"\xed\xbf\xbf\"")), + "utf8-scan FAILED: encoded low surrogate not flagged"); + + // Above U+10FFFF + assert(hasInvalidUTF8(b("\"\xf4\x90\x80\x80\"")), + "utf8-scan FAILED: 0xF4 0x90 (above U+10FFFF) not flagged"); + assert(hasInvalidUTF8(b("\"\xf5\x80\x80\x80\"")), + "utf8-scan FAILED: 0xF5 lead byte not flagged"); + assert(hasInvalidUTF8(b("\"\xff\"")), + "utf8-scan FAILED: 0xFF byte not flagged"); + + // Bare continuation as first byte of input + assert(hasInvalidUTF8(b("\x80")), + "utf8-scan FAILED: leading stray continuation not flagged"); + + // Well-formed input must pass, including range boundaries + assert(!hasInvalidUTF8(b(`{"a":1,"s":"plain ascii"}`)), + "utf8-scan FAILED: ASCII wrongly flagged"); + assert(!hasInvalidUTF8(b("\"\xc3\xa9\"")), + "utf8-scan FAILED: 2-byte e-acute wrongly flagged"); + assert(!hasInvalidUTF8(b("\"\xe2\x82\xac\"")), + "utf8-scan FAILED: 3-byte euro sign wrongly flagged"); + assert(!hasInvalidUTF8(b("\"\xf0\x9f\x98\x80\"")), + "utf8-scan FAILED: 4-byte emoji wrongly flagged"); + assert(!hasInvalidUTF8(b("\"\xed\x9f\xbf\"")), + "utf8-scan FAILED: U+D7FF (below surrogates) wrongly flagged"); + assert(!hasInvalidUTF8(b("\"\xee\x80\x80\"")), + "utf8-scan FAILED: U+E000 (above surrogates) wrongly flagged"); + assert(!hasInvalidUTF8(b("\"\xf4\x8f\xbf\xbf\"")), + "utf8-scan FAILED: U+10FFFF (top of range) wrongly flagged"); +} + +// ── Strict token-lexeme scan tests ── +// RFC 8259 number grammar (no leading zeros, no attached junk) and +// exact-spelling literals: parseJSON accepts all of these laxly, so +// the raw scan must reject them. +unittest +{ + import baionstd.types : StdError; + + // Leading-zero numbers — the agreement-fuzzer counterexamples + assert(scanStrictTokens(`0635`).get == StdError.invalidNumber, + "strict-token FAILED: 0635 not flagged"); + assert(scanStrictTokens(`-004`).get == StdError.invalidNumber, + "strict-token FAILED: -004 not flagged"); + assert(scanStrictTokens(`01`).get == StdError.invalidNumber, + "strict-token FAILED: 01 not flagged"); + assert(scanStrictTokens(`03000000000000004`).get == StdError.invalidNumber, + "strict-token FAILED: 03000000000000004 not flagged"); + assert(scanStrictTokens(`{"a":01}`).get == StdError.invalidNumber, + "strict-token FAILED: nested leading zero not flagged"); + + // Junk attached to a bare number + assert(scanStrictTokens(`2-`).get == StdError.invalidNumber, + "strict-token FAILED: 2- not flagged"); + assert(scanStrictTokens(`1-`).get == StdError.invalidNumber, + "strict-token FAILED: 1- not flagged"); + assert(scanStrictTokens(`77-7957`).get == StdError.invalidNumber, + "strict-token FAILED: 77-7957 not flagged"); + assert(scanStrictTokens(`[1, 2-]`).get == StdError.invalidNumber, + "strict-token FAILED: nested attached junk not flagged"); + + // Case-variant literals + assert(scanStrictTokens(`nuLl`).get == StdError.invalidLiteral, + "strict-token FAILED: nuLl not flagged"); + assert(scanStrictTokens(`falSe`).get == StdError.invalidLiteral, + "strict-token FAILED: falSe not flagged"); + assert(scanStrictTokens(`tRue`).get == StdError.invalidLiteral, + "strict-token FAILED: tRue not flagged"); + assert(scanStrictTokens(`{"a":[TRUE]}`).get == StdError.invalidLiteral, + "strict-token FAILED: nested TRUE not flagged"); + + // Legal tokens must pass untouched + assert(scanStrictTokens(`0`).isNull, + "strict-token FAILED: bare 0 wrongly flagged"); + assert(scanStrictTokens(`-0.001`).isNull, + "strict-token FAILED: -0.001 wrongly flagged"); + assert(scanStrictTokens(`10`).isNull, + "strict-token FAILED: 10 wrongly flagged"); + assert(scanStrictTokens(`{"a":0.5,"b":[null,true,false]}`).isNull, + "strict-token FAILED: legal document wrongly flagged"); + // Exponent forms are grammar-valid HERE (domain scan owns them) + assert(scanStrictTokens(`1e21`).isNull, + "strict-token FAILED: exponent form flagged by the wrong scanner"); + // Digits and keyword-looking text inside strings are not tokens + assert(scanStrictTokens(`{"s":"0635 nuLl 2-"}`).isNull, + "strict-token FAILED: in-string text wrongly flagged"); +} diff --git a/d/tools/canon_hash_main.d b/d/tools/canon_hash_main.d index 79facf3..5ce3781 100644 --- a/d/tools/canon_hash_main.d +++ b/d/tools/canon_hash_main.d @@ -4,9 +4,13 @@ // Reads UTF-8 JSON on stdin, writes the SHA-256 of its canonical form // (sorted keys, no whitespace, shortest-round-trip floats) as lowercase // hex + newline. Exit 0 on success, nonzero on parse error, on a -// duplicate object key (decoded names) at any depth, or when stdin is -// not exactly one JSON document (empty input, trailing data, -// concatenated documents, trailing comma). +// duplicate object key (decoded names) at any depth, on a raw control +// byte between tokens that is not JSON whitespace, on invalid UTF-8 in +// the raw bytes (RFC 3629), on a number token violating the RFC 8259 +// grammar (leading zero, attached junk), on a literal that is not +// exactly null/true/false, or when stdin is not exactly one JSON +// document (empty input, trailing data, concatenated documents, +// trailing comma). module tools.canon_hash_main; @@ -15,7 +19,8 @@ import std.json : parseJSON; import std.stdio : stdin, stdout, stderr; import baionstd.canonical_json : canonicalizeJSON, hasDuplicateKeys, - hasUnsupportedNumber, scanSingleDocument; + hasInvalidUTF8, hasUnsupportedControl, hasUnsupportedNumber, + scanSingleDocument, scanStrictTokens; import baionstd.hash : sha256Hex; import baionstd.types : StdError, errorMessage; @@ -28,8 +33,15 @@ int main() string canonical; try { - // WHY validate + parse in one step: parseJSON rejects both malformed - // JSON and invalid UTF-8, which is exactly the CLI's error contract. + // UTF-8 scan on the RAW BYTES, before any parse — parseJSON + // (measured, dmd 2.112.0) passes invalid UTF-8 through untouched, + // and every later scanner assumes RFC 3629-valid input. + if (hasInvalidUTF8(buf[])) + { + stderr.writeln("baion_canon_hash: ", errorMessage(StdError.invalidUTF8)); + return 1; + } + auto j = parseJSON(cast(const(char)[]) buf[]); // Single-document scan on the RAW input — parseJSON stops at the @@ -45,6 +57,28 @@ int main() return 1; } + // Strict token-lexeme scan on the RAW input — parseJSON accepts + // leading-zero numbers, junk attached to a bare number (`2-` + // hashes as 2) and CASE-INSENSITIVE null/true/false spellings; + // none of that survives into the parsed value. + auto tokErr = scanStrictTokens(cast(const(char)[]) buf[]); + if (!tokErr.isNull) + { + stderr.writeln("baion_canon_hash: ", errorMessage(tokErr.get)); + return 1; + } + + // Control-byte scan on the RAW input — parseJSON's inter-token + // whitespace skip (isWhite) also accepts 0x0B/0x0C, which RFC + // 8259 whitespace excludes; the parsed value carries no trace. + // (In-string raw controls never reach here: parseJSON already + // rejected them as "Illegal control character".) + if (hasUnsupportedControl(cast(const(char)[]) buf[])) + { + stderr.writeln("baion_canon_hash: ", errorMessage(StdError.unsupportedControl)); + return 1; + } + // Duplicate-key scan on the RAW input — parseJSON's associative // array has already deduplicated silently (keeps last), so the // check cannot run on the parsed value. Runs after parseJSON so diff --git a/go/canonical_json.go b/go/canonical_json.go index ffecd48..e0d4710 100644 --- a/go/canonical_json.go +++ b/go/canonical_json.go @@ -17,6 +17,7 @@ import ( "sort" "strconv" "strings" + "unicode/utf8" ) // ErrEmbeddedNUL rejects inputs whose strings contain U+0000. @@ -48,6 +49,17 @@ var ErrDuplicateKey = errors.New("duplicate object key") // `100` is in-domain while `1e2` is rejected even though the values are equal. var ErrUnsupportedNumber = errors.New("unsupported number outside the cross-lineage numeric domain") +// ErrInvalidUTF8 rejects inputs whose raw bytes are not well-formed UTF-8. +// +// CROSS-LINEAGE CONTRACT: every language implementation uniformly REJECTS +// input that is not valid UTF-8 — Go's encoding/json alone silently replaces +// invalid byte sequences with U+FFFD, which would let Go hash input no +// sibling accepts (and hash it to bytes no sibling would ever produce): a +// silent cross-lineage collision split. The check must run on the RAW input +// bytes, before decoding, because the decoder's U+FFFD substitution destroys +// the evidence. +var ErrInvalidUTF8 = errors.New("invalid or unsupported UTF-8 in input") + // ErrLoneSurrogate rejects inputs containing an unpaired UTF-16 surrogate // escape (\uD800–\uDFFF without its partner) inside a JSON string. // @@ -57,6 +69,23 @@ var ErrUnsupportedNumber = errors.New("unsupported number outside the cross-line // detected on the RAW bytes — after decode the evidence is destroyed. var ErrLoneSurrogate = errors.New("unsupported unpaired surrogate escape in string") +// CheckValidUTF8 verifies the raw input bytes are well-formed UTF-8 and +// returns ErrInvalidUTF8 otherwise. +// +// This check MUST run on the raw input bytes, before any decode: Go's +// encoding/json accepts invalid UTF-8 and silently substitutes U+FFFD, so a +// post-decode check can never see the bad bytes. utf8.Valid uses strict UTF-8 +// decoding — it rejects overlong encodings, truncated sequences, stray +// continuation bytes, AND UTF-8-encoded surrogate code points (0xED 0xA0 0x80 +// decodes as RuneError with size 1), so no separate surrogate-byte check is +// needed here (escaped \uD800 surrogates are CheckNoLoneSurrogates' job). +func CheckValidUTF8(data []byte) error { + if !utf8.Valid(data) { + return ErrInvalidUTF8 + } + return nil +} + // CheckNoDuplicateKeys scans raw JSON input for duplicate object member names // at any depth and returns ErrDuplicateKey if one is found. // diff --git a/go/canonical_json_test.go b/go/canonical_json_test.go index 64b26c0..2addb09 100644 --- a/go/canonical_json_test.go +++ b/go/canonical_json_test.go @@ -313,6 +313,56 @@ func TestCanonicalJSON_LoneSurrogateRejection(t *testing.T) { } } +// Invalid-UTF-8 contract: the raw input bytes must be well-formed UTF-8. +// encoding/json alone would silently substitute U+FFFD for bad bytes and hash +// input every sibling lineage rejects, so CheckValidUTF8 must catch every +// invalid class on the raw bytes — including UTF-8-ENCODED surrogates (0xED +// 0xA0 0x80), which utf8.Valid rejects via strict decoding (verified here so a +// toolchain regression would be caught, not silently absorbed). +// Payloads are built with []byte literals — never raw invalid bytes in source. +func TestCanonicalJSON_InvalidUTF8Rejection(t *testing.T) { + rejects := []struct { + name string + input []byte + }{ + {"truncated_lead_in_key", append([]byte(`{"`), append([]byte{0xe0}, []byte(`a":[]}`)...)...)}, + {"stray_continuation_in_string", []byte{'"', 'a', 0x85, 'b', '"'}}, + {"overlong_slash", []byte{'"', 0xc0, 0xaf, '"'}}, + {"encoded_surrogate_d800", []byte{'"', 0xed, 0xa0, 0x80, '"'}}, + {"encoded_surrogate_dfff", []byte{'"', 0xed, 0xbf, 0xbf, '"'}}, + {"truncated_two_byte_at_eof", []byte{'"', 0xc3, '"'}}, + {"bare_ff", []byte{'"', 0xff, '"'}}, + {"truncated_four_byte", []byte{'"', 0xf0, 0x9f, 0x98, '"'}}, + } + for _, tt := range rejects { + t.Run("reject_"+tt.name, func(t *testing.T) { + if err := CheckValidUTF8(tt.input); err != ErrInvalidUTF8 { + t.Errorf("input % x: want ErrInvalidUTF8, got %v", tt.input, err) + } + }) + } + + accepts := []struct { + name string + input []byte + }{ + {"ascii", []byte(`{"a":1}`)}, + {"two_byte_e_acute", []byte{'"', 0xc3, 0xa9, '"'}}, + {"three_byte_euro", []byte{'"', 0xe2, 0x82, 0xac, '"'}}, + {"four_byte_emoji", []byte{'"', 0xf0, 0x9f, 0x98, 0x80, '"'}}, + {"just_below_surrogates_ud7ff", []byte{'"', 0xed, 0x9f, 0xbf, '"'}}, + {"just_above_surrogates_ue000", []byte{'"', 0xee, 0x80, 0x80, '"'}}, + {"empty", []byte{}}, + } + for _, tt := range accepts { + t.Run("accept_"+tt.name, func(t *testing.T) { + if err := CheckValidUTF8(tt.input); err != nil { + t.Errorf("input % x: want nil, got %v", tt.input, err) + } + }) + } +} + // Negative-zero contract: any zero value emits exactly "0" (RFC 8785 / ES // ToString) — both the fraction spelling -0.0 (float path) and the integer // spelling -0 (int64 path). diff --git a/go/cmd/baion_canon_hash/main.go b/go/cmd/baion_canon_hash/main.go index 7a3fa56..0f52820 100644 --- a/go/cmd/baion_canon_hash/main.go +++ b/go/cmd/baion_canon_hash/main.go @@ -26,6 +26,14 @@ func main() { os.Exit(1) } + // UTF-8 validation must run FIRST, on the raw bytes: encoding/json accepts + // invalid UTF-8 and silently substitutes U+FFFD, which would hash input + // every sibling lineage rejects — a silent cross-lineage collision split. + if err := baionstd.CheckValidUTF8(input); err != nil { + fmt.Fprintf(os.Stderr, "baion_canon_hash: reject: invalid UTF-8 in input (%v)\n", err) + os.Exit(1) + } + // Duplicate-key detection needs the RAW bytes: decoding into // map[string]interface{} keeps only one member per name and destroys the // duplicate, so this contract check must run before the decode below. diff --git a/haskell/app/Main.hs b/haskell/app/Main.hs index 6f2e2cf..d284d94 100644 --- a/haskell/app/Main.hs +++ b/haskell/app/Main.hs @@ -8,6 +8,7 @@ module Main (main) where import Baion.STD.CanonicalJson ( canonicalizeJsonChecked, + checkControlBytes, checkNoDuplicateKeys, checkNumberDomain, ) @@ -22,30 +23,41 @@ import System.IO (hPutStrLn, stderr) main :: IO () main = do input <- BS.getContents - -- STRICT single-document contract: aeson >= 2.2's eitherDecodeStrict' - -- (Data.Aeson.Decoding path) requires exactly one complete JSON - -- document with the whole input consumed — trailing garbage, a second - -- document, a trailing comma, and empty input all fail here (only - -- trailing whitespace is allowed). Suite tests 15-19 pin this so an - -- aeson behavior change cannot silently relax it. - case A.eitherDecodeStrict' input :: Either String A.Value of + -- Raw-control-byte gate runs BEFORE the decode: it is purely + -- byte-level (safe on malformed input), and aeson's current lexer + -- would otherwise reject the same inputs first with a parser-internal + -- message — running the contract check first keeps the error text + -- cross-lineage uniform and keeps the contract off a dependency's + -- strictness (an aeson upgrade cannot silently relax it). + case checkControlBytes input of Left err -> do - hPutStrLn stderr ("baion-canon-hash: parse error: " ++ err) + hPutStrLn stderr ("baion-canon-hash: " ++ err) exitFailure - Right v -> - -- Duplicate-key and number-domain checks both run on the RAW - -- bytes (aeson's KeyMap drops duplicates at parse, and Scientific - -- erases the lexical 100-vs-1e2 distinction the number contract - -- is defined over); then checked canonicalization rejects any - -- string containing U+0000 (aeson preserves NUL in Text, so the - -- CLI would otherwise pass it through to the digest). - case checkNoDuplicateKeys input - >> checkNumberDomain input - >> canonicalizeJsonChecked v of + Right () -> + -- STRICT single-document contract: aeson >= 2.2's eitherDecodeStrict' + -- (Data.Aeson.Decoding path) requires exactly one complete JSON + -- document with the whole input consumed — trailing garbage, a second + -- document, a trailing comma, and empty input all fail here (only + -- trailing whitespace is allowed). Suite tests 15-19 pin this so an + -- aeson behavior change cannot silently relax it. + case A.eitherDecodeStrict' input :: Either String A.Value of Left err -> do - hPutStrLn stderr ("baion-canon-hash: " ++ err) + hPutStrLn stderr ("baion-canon-hash: parse error: " ++ err) exitFailure - Right canonical -> - -- CROSS-LINEAGE CONTRACT: digest is over the canonical string's - -- UTF-8 bytes, matching the other lineages' canonical-bytes hash. - putStrLn (sha256HexBytes (TE.encodeUtf8 (T.pack canonical))) + Right v -> + -- Duplicate-key and number-domain checks both run on the RAW + -- bytes (aeson's KeyMap drops duplicates at parse, and Scientific + -- erases the lexical 100-vs-1e2 distinction the number contract + -- is defined over); then checked canonicalization rejects any + -- string containing U+0000 (aeson preserves NUL in Text, so the + -- CLI would otherwise pass it through to the digest). + case checkNoDuplicateKeys input + >> checkNumberDomain input + >> canonicalizeJsonChecked v of + Left err -> do + hPutStrLn stderr ("baion-canon-hash: " ++ err) + exitFailure + Right canonical -> + -- CROSS-LINEAGE CONTRACT: digest is over the canonical string's + -- UTF-8 bytes, matching the other lineages' canonical-bytes hash. + putStrLn (sha256HexBytes (TE.encodeUtf8 (T.pack canonical))) diff --git a/haskell/src/Baion/STD/CanonicalJson.hs b/haskell/src/Baion/STD/CanonicalJson.hs index 0dccd81..09397c5 100644 --- a/haskell/src/Baion/STD/CanonicalJson.hs +++ b/haskell/src/Baion/STD/CanonicalJson.hs @@ -7,6 +7,7 @@ module Baion.STD.CanonicalJson ( canonicalizeJson, canonicalizeJsonChecked, + checkControlBytes, checkNoDuplicateKeys, checkNumberDomain, writeJsonString, @@ -23,7 +24,8 @@ import Data.Aeson.Decoding.Tokens import qualified Data.Aeson.Key as AK import qualified Data.Aeson.KeyMap as AKM import qualified Data.ByteString as BS -import Data.Char (intToDigit, ord) +import Data.Char (ord) +import Data.List (foldl') import qualified Data.Map.Strict as Map import qualified Data.Scientific as S import qualified Data.Set as Set @@ -94,6 +96,58 @@ checkNoDuplicateKeys bs = case scanTokens (bsToTokens bs) of scanRecord _ (TkRecordEnd k) = Right (Just k) scanRecord _ (TkRecordErr _) = Right Nothing +-- | Reject raw control bytes (0x00-0x1F) per RFC 8259: inside string +-- literals every control byte must be escape-spelled; between tokens +-- only TAB/LF/CR (0x09/0x0A/0x0D, plus 0x20 which is not a control +-- byte) are legal whitespace. aeson >= 2.2's Decoding lexer happens to +-- enforce both today, but that is an implementation detail of a +-- dependency — this pass pins the contract on the RAW bytes so an +-- aeson upgrade (or a library caller bypassing the CLI's decoder) +-- cannot silently relax it, and the rejection carries the uniform +-- cross-lineage error text instead of a parser-internal message. +-- Sibling of 'checkNumberDomain': same escape-aware string-skipping +-- walk. Escape TEXT like backslash-t or backslash-u001F is bytes +-- 0x5C 0x74 / 0x5C 0x75..., all >= 0x20 — a byte-level check cannot +-- false-positive on it. Multi-byte UTF-8 continuation bytes are +-- >= 0x80, so byte-wise scanning is safe. +-- CROSS-LINEAGE CONTRACT: all 7 lineages reject raw control bytes +-- identically (C++/Rust/Go/D already did; Haskell relied on aeson). +checkControlBytes :: BS.ByteString -> Either String () +checkControlBytes = goTop + where + goTop bs = case BS.uncons bs of + Nothing -> Right () + Just (c, rest) + | c == 0x22 -> inString rest -- '"' + | c < 0x20 && c /= 0x09 && c /= 0x0a && c /= 0x0d -> + Left (controlErr "between tokens" c) + | otherwise -> goTop rest + + -- Inside a string literal every byte < 0x20 is illegal — including + -- the byte after a backslash (no valid escape character is a + -- control byte), so the escaped byte is checked before being + -- consumed. Unterminated strings fall off the end silently: the + -- CLI's strict decoder owns malformed-input errors. + inString bs = case BS.uncons bs of + Nothing -> Right () + Just (c, rest) + | c < 0x20 -> Left (controlErr "inside a string literal" c) + | c == 0x5c -> case BS.uncons rest of -- '\\' + Nothing -> Right () + Just (c2, rest2) + | c2 < 0x20 -> Left (controlErr "inside a string literal" c2) + | otherwise -> inString rest2 + | c == 0x22 -> goTop rest -- closing '"' + | otherwise -> inString rest + + controlErr loc c = + printf + ( "unsupported raw control byte 0x%02X %s: control characters" + ++ " must be escaped (RFC 8259 / cross-lineage contract)" + ) + (fromIntegral c :: Int) + (loc :: String) + -- | Reject number tokens outside the cross-lineage number domain. -- This must scan the RAW input: aeson decodes every number spelling -- into 'S.Scientific', so by the time a 'A.Value' exists the lexical @@ -198,11 +252,14 @@ canonicalizeValue :: A.Value -> String canonicalizeValue A.Null = "null" canonicalizeValue (A.Bool True) = "true" canonicalizeValue (A.Bool False) = "false" -canonicalizeValue (A.Number n) - | S.isInteger n = case S.floatingOrInteger n of - Right i -> show (i :: Integer) - Left d -> showDouble d - | otherwise = showDouble (S.toRealFloat n) +-- All numbers render through DOUBLE semantics — even exact-integer +-- Scientifics. The other six lineages hold only a double by this +-- point, so an exact-Integer fast path here would diverge for +-- fraction tokens whose exact value is an unrepresentable integer +-- (e.g. 9007199254740993.0, which every double lineage renders as +-- 9007199254740992). Gated integer tokens (|i| <= 2^53) are exactly +-- representable, so this path prints them identically to `show`. +canonicalizeValue (A.Number n) = showDouble (S.toRealFloat n) canonicalizeValue (A.String t) = writeJsonString (T.unpack t) canonicalizeValue (A.Array arr) = "[" ++ commaJoin (map canonicalizeValue (V.toList arr)) ++ "]" @@ -225,9 +282,20 @@ canonicalizeValue (A.Object obj) = -- shortest-search loop switched to scientific notation below 0.01 -- and near 1e21 ("1.0e-3"), which diverged from the reference for -- every fraction in those bands (fixed 2026-07-15). --- 'Numeric.floatToDigits' 10 yields exactly the minimal (shortest --- round-tripping) digit string ES-262 specifies, with no trailing --- zeros, so reassembly is pure positional bookkeeping. +-- 'Numeric.floatToDigits' 10 yields a round-tripping digit string +-- with no trailing zeros, but NOT always the minimal one ES-262 +-- specifies: GHC's Burger–Dybvig loop compares against the rounding +-- interval with strict inequalities regardless of mantissa parity, +-- so when the shortest decimal sits EXACTLY on an interval boundary +-- that IEEE round-half-even makes inclusive (even mantissa), it +-- emits one digit too many. Found by the agreement fuzzer as a +-- seven-lineage hash split (2026-07-15): 65219416364867774.9377591 +-- parses to 0x436CF696D61C5C18, whose shortest spelling +-- 6521941636486778e1 is the exact upper midpoint — floatToDigits +-- returned the 17-digit exact integer 65219416364867776 while every +-- other lineage emitted 65219416364867780. 'shortenScaled' below +-- greedily drops trailing digits while the value still round-trips, +-- restoring the ES-262 minimum. -- CROSS-LINEAGE CONTRACT: byte-identical to ECMAScript ToString for -- doubles inside the gated fraction domain [1e-6, 1e21). showDouble :: Double -> String @@ -239,11 +307,45 @@ showDouble d -- ES-262 notation: value = D × 10^(n − k) with D the minimal digit -- string and k = length D. floatToDigits 10 x = (digits, e) means --- x = 0.D × 10^e = D × 10^(e − k), so n = e directly. +-- x = 0.D × 10^e = D × 10^(e − k); after shortening we track the +-- value as m × 10^scale, so n = scale + length (show m). positiveToString :: Double -> String positiveToString d = - let (ds, n) = floatToDigits 10 d - in assembleEs262 (map intToDigit ds) n + let (ds0, n0) = floatToDigits 10 d + m0 = foldl' (\acc x -> acc * 10 + toInteger x) 0 ds0 + (m, scale) = shortenScaled d m0 (n0 - length ds0) + ds = show m + in assembleEs262 ds (scale + length ds) + +-- | Greedy ES-262 shortening pass over (m, scale) with value +-- m × 10^scale == d exactly round-tripped. Each step tries the two +-- one-digit-shorter candidates (truncate, truncate+1); a candidate +-- survives only if it still converts to exactly d ('S.toRealFloat' +-- is correctly rounded). When both survive, ES-262 §7.1.12.1 picks +-- the spelling closest to the value, breaking a tie toward the even +-- significand. Carry (q+1 rolling to a power of 10, e.g. 999 -> 100) +-- is safe: digits are recomputed from the Integer each round. +shortenScaled :: Double -> Integer -> Int -> (Integer, Int) +shortenScaled d = go + where + go m scale + | m < 10 = (m, scale) + | otherwise = + let (q, r) = m `divMod` 10 + cands = if r == 0 then [q] else [q, q + 1] + in case [c | c <- cands, sciValue c (scale + 1) == d] of + [] -> (m, scale) + [c] -> go c (scale + 1) + cs -> go (closerToD cs (scale + 1)) (scale + 1) + sciValue c e = S.toRealFloat (S.scientific c e) :: Double + closerToD [a, b] e + | dist a < dist b = a + | dist b < dist a = b + | even a = a + | otherwise = b + where + dist c = abs (fromInteger c * (10 ^^ e) - toRational d) + closerToD cs _ = head cs -- unreachable: cands has at most 2 members assembleEs262 :: String -> Int -> String assembleEs262 ds n diff --git a/haskell/test/ConformanceTest.hs b/haskell/test/ConformanceTest.hs index 805baa5..a8a188e 100644 --- a/haskell/test/ConformanceTest.hs +++ b/haskell/test/ConformanceTest.hs @@ -9,9 +9,11 @@ import Baion.STD import qualified Data.Aeson as A import qualified Data.Aeson.Key as AK import qualified Data.Aeson.KeyMap as AKM +import qualified Data.ByteString as BS import qualified Data.ByteString.Char8 as BSC import qualified Data.Text as T import qualified Data.Text.Encoding as TE +import Data.Word (Word8) import System.Directory (doesFileExist) import Test.Tasty import Test.Tasty.HUnit @@ -143,9 +145,144 @@ conformanceTests = assertCanonicalizes "{\"x\":0.1}" "{\"x\":0.1}" assertCanonicalizes "{\"x\":123.456}" "{\"x\":123.456}" assertCanonicalizes "{\"x\":1.5}" "{\"x\":1.5}" + assertCanonicalizes "{\"x\":1.0}" "{\"x\":1}", + testCase "Test 30: raw control byte inside a string literal rejected" $ do + assertControlRejected (rawInString 0x00) + assertControlRejected (rawInString 0x01) + assertControlRejected (rawInString 0x09) -- raw TAB + assertControlRejected (rawInString 0x0a) -- raw LF + assertControlRejected (rawInString 0x1e) + assertControlRejected (rawInString 0x1f) + -- Pin the finding that aeson's Decoding lexer ALSO rejects this + -- today — if an aeson upgrade relaxes it, this test still holds + -- via checkControlBytes, but the pin documents the redundancy. + case A.eitherDecodeStrict' (rawInString 0x09) :: Either String A.Value of + Left _ -> return () + Right _ -> + assertFailure + "aeson newly ACCEPTS raw TAB in a string; checkControlBytes is now the only guard", + testCase "Test 31: raw control byte between tokens rejected" $ do + assertControlRejected (rawBetweenTokens 0x00) + assertControlRejected (rawBetweenTokens 0x02) + assertControlRejected (rawBetweenTokens 0x0b) -- VT is not JSON ws + assertControlRejected (rawBetweenTokens 0x0c) -- FF is not JSON ws + assertControlRejected (rawBetweenTokens 0x1f), + testCase "Test 32: legal whitespace between tokens accepted" $ do + assertControlAccepted (rawBetweenTokens 0x09) -- TAB + assertControlAccepted (rawBetweenTokens 0x0a) -- LF + assertControlAccepted (rawBetweenTokens 0x0d) -- CR + assertControlAccepted (rawBetweenTokens 0x20) -- space + assertControlAccepted (BSC.pack "\n{\"a\":1}\r\n"), + testCase "Test 33: escape TEXT passes; escape-aware string skipping" $ do + -- Two-char escape text backslash-t / six-char backslash-u001f + -- are bytes 0x5C 0x74 / 0x5C 0x75... — never control bytes. + assertControlAccepted (BSC.pack "{\"s\":\"a\\tb\"}") + assertControlAccepted (BSC.pack "{\"s\":\"a\\u001fb\"}") + assertControlAccepted (BSC.pack "{\"s\":\"a\\u0000b\"}") + -- Escaped quote must not end the string scan early... + assertControlAccepted (BSC.pack "{\"s\":\"a\\\"b\",\"t\":\"c\"}") + -- ...and a raw control byte AFTER an escaped quote is still + -- inside the string ({"s":"a\"b"}). + assertControlRejected + ( BS.pack + [0x7b, 0x22, 0x73, 0x22, 0x3a, 0x22, 0x61, 0x5c, 0x22, 0x09, 0x62, 0x22, 0x7d] + ) + -- A raw control byte as the escaped byte itself (backslash + -- immediately followed by raw TAB) is rejected, not skipped. + assertControlRejected + ( BS.pack + [0x7b, 0x22, 0x73, 0x22, 0x3a, 0x22, 0x61, 0x5c, 0x09, 0x62, 0x22, 0x7d] + ), + -- Tests 34-37: ES-262 shortest-digits contract above 2^53, where + -- the exact integer value of a double and its shortest spelling + -- diverge. GHC's floatToDigits misses the even-mantissa inclusive + -- boundary (emits 17 digits at exact midpoints); shortenScaled in + -- CanonicalJson.hs restores the minimum. Expected strings/hashes + -- cross-checked against rust/bin/baion_canon_hash 2026-07-15. + testCase "Test 34: even-mantissa midpoint uses shortest spelling (fuzzer hash split)" $ do + -- double 0x436CF696D61C5C18: exact 65219416364867776, ES-262 + -- shortest 6521941636486778e1 = 65219416364867780. + assertCanonicalizes "65219416364867774.9377591" "65219416364867780" + assertHashes + "65219416364867774.9377591" + "077d9fcc047c90f56dc97fc7dc513bbeb8832b6c2266440f8fb5b7bf958c6596", + testCase "Test 35: 1e20-adjacent fraction keeps six-lineage hash" $ do + assertCanonicalizes + "{\"x\":100000000000000000000.5}" + "{\"x\":100000000000000000000}" + assertHashes + "{\"x\":100000000000000000000.5}" + "356acd219b8c369fc389513fb5c3f9fc2977fff3c432bab9df5b1e3a2800b072", + testCase "Test 36: 2^53-adjacent integer-valued doubles render the DOUBLE value" $ do + -- 9007199254740993 is not representable; the double is ...992. + assertCanonicalizes "9007199254740993.0" "9007199254740992" + assertHashes + "9007199254740993.0" + "c681da39d7273a6a24c15c9cac3a75526ff2ecf8ba4ee60346a0c70c8163bdb2" + assertCanonicalizes "9007199254740994.5" "9007199254740994" + assertHashes + "9007199254740994.5" + "25aa68783313802627958889943e895749ac4c0c7469b2a305cd450a12120768", + testCase "Test 37: big integer-valued doubles cross-checked against rust lineage" $ do + assertCanonicalizes "5000000000000000000.7" "5000000000000000000" + assertHashes + "5000000000000000000.7" + "eebc2ee21907fb949e3a007794ca384b1c12a65088f6df41a31687b2a07f3bb8" + assertCanonicalizes "314159265358979323.846" "314159265358979300" + assertHashes + "314159265358979323.846" + "8671760432f785f41764e0ee0f2282bc1cdc84cc677ec41e207e750f512d1a06" + -- Small integer-valued floats must be untouched by the + -- shortening pass (corpus-pinned spellings). assertCanonicalizes "{\"x\":1.0}" "{\"x\":1}" + assertCanonicalizes "{\"x\":-0.0}" "{\"x\":0}" ] +-- Full-pipeline hash pin: decode, canonicalize, SHA-256 the UTF-8 +-- bytes — must equal the six-lineage digest for the same input. +assertHashes :: String -> String -> Assertion +assertHashes raw expectedHex = + case A.eitherDecodeStrict' (BSC.pack raw) :: Either String A.Value of + Left err -> assertFailure ("fixture must parse: " ++ err) + Right v -> canonicalSha256Hex (canonicalizeJson v) @?= expectedHex + +-- Raw-control-byte payload builders (tests 30-33). HAZARD: the raw +-- byte is assembled at runtime via BS.pack — never pasted into a +-- source literal, so no editor/toolchain can normalize it away. + +-- | {"s":"ab"} with byte b spliced raw inside the string literal. +rawInString :: Word8 -> BS.ByteString +rawInString b = + BS.pack [0x7b, 0x22, 0x73, 0x22, 0x3a, 0x22, 0x61, b, 0x62, 0x22, 0x7d] + +-- | {"a":1,"b":2} with byte b spliced raw between tokens. +rawBetweenTokens :: Word8 -> BS.ByteString +rawBetweenTokens b = + BS.pack + [0x7b, 0x22, 0x61, 0x22, 0x3a, 0x31, 0x2c, b, 0x22, 0x62, 0x22, 0x3a, 0x32, 0x7d] + +-- Raw-control-byte contract (tests 30-33): lexical check over raw +-- bytes; error text must carry both "unsupported" and "control" so +-- pipeline greps can classify the failure. +assertControlRejected :: BS.ByteString -> Assertion +assertControlRejected raw = + case checkControlBytes raw of + Left err -> + assertBool + ("error message must mention unsupported control byte, got: " ++ err) + ( T.isInfixOf "unsupported" (T.pack err) + && T.isInfixOf "control" (T.pack err) + ) + Right () -> + assertFailure ("expected control-byte rejection for: " ++ show raw) + +assertControlAccepted :: BS.ByteString -> Assertion +assertControlAccepted raw = + case checkControlBytes raw of + Left err -> + assertFailure ("expected acceptance for " ++ show raw ++ ", got: " ++ err) + Right () -> return () + -- STRICT single-document contract (tests 15-19): aeson >= 2.2's -- eitherDecodeStrict' must consume the whole input as exactly one -- JSON document (trailing whitespace only). Pinned here so an aeson diff --git a/ocaml/cli/baion_canon_hash.ml b/ocaml/cli/baion_canon_hash.ml index 12ced11..d172bae 100644 --- a/ocaml/cli/baion_canon_hash.ml +++ b/ocaml/cli/baion_canon_hash.ml @@ -7,6 +7,30 @@ let () = let input = In_channel.input_all In_channel.stdin in + (* UTF-8 well-formedness runs first, on the RAW bytes: yojson passes + invalid byte sequences through string literals verbatim, and every + later pass assumes it is walking sound UTF-8. *) + (match Baionstd_public.Canonical_json.check_utf8 input with + | exception Baionstd_public.Canonical_json.Invalid_utf8 msg -> + prerr_endline ("baion_canon_hash: invalid input: " ^ msg); + exit 1 + | () -> ()); + (* Strict token grammar runs on the RAW bytes before parsing: yojson + accepts unquoted object keys, comments, and NaN/Infinity literals, + none of which are recoverable after parse. *) + (match Baionstd_public.Canonical_json.check_strict_tokens input with + | exception Baionstd_public.Canonical_json.Nonstandard_token msg -> + prerr_endline ("baion_canon_hash: invalid input: " ^ msg); + exit 1 + | () -> ()); + (* Raw-control-byte enforcement runs on the RAW bytes before parsing: + yojson accepts an unescaped 0x01-0x1F byte inside a string literal, + and by parse time it is indistinguishable from the escaped form. *) + (match Baionstd_public.Canonical_json.check_no_raw_control_chars input with + | exception Baionstd_public.Canonical_json.Control_char_rejected msg -> + prerr_endline ("baion_canon_hash: invalid input: " ^ msg); + exit 1 + | () -> ()); (* Number-domain enforcement runs on the RAW bytes before parsing: yojson normalizes 1e2 to the same value as 100, so exponent spelling is only visible lexically. *) diff --git a/ocaml/lib/canonical_json.ml b/ocaml/lib/canonical_json.ml index 3ef9fa4..443995b 100644 --- a/ocaml/lib/canonical_json.ml +++ b/ocaml/lib/canonical_json.ml @@ -203,6 +203,203 @@ let check_no_lone_surrogates (raw : string) : unit = end done +(** Raised when the raw input carries an unescaped control byte + (0x00-0x1F) inside a string literal, or a non-whitespace control + byte between tokens. *) +exception Control_char_rejected of string + +(* CROSS-LINEAGE CONTRACT: raw (unescaped) control bytes must be + rejected by all seven lineage libraries. RFC 8259 §7 requires + control characters inside string literals to be escaped, and §2 + allows only TAB/LF/CR/space as insignificant whitespace between + tokens — but yojson accepts a raw 0x01-0x1F byte inside a string + literal (e.g. a real TAB in {"s":"ab"}) where C++/Rust/Go/D + refuse. By parse time that raw byte is indistinguishable from the + escaped spelling, so this is a LEXICAL pass over the raw input, + sibling of reject_unsupported_numbers and using the same + skip-strings walk — except here the bytes INSIDE string literals + are the ones inspected. Escaped forms (backslash-t, backslash-u001f) + never trip this: they are ordinary printable bytes in the raw text. *) +let check_no_raw_control_chars (raw : string) : unit = + let len = String.length raw in + let reject_in_string c = + raise + (Control_char_rejected + (Printf.sprintf + "unsupported raw control character (0x%02x) in string literal" + (Char.code c))) + in + let i = ref 0 in + while !i < len do + let c = raw.[!i] in + if c = '"' then begin + incr i; + let closed = ref false in + while (not !closed) && !i < len do + let c = raw.[!i] in + if Char.code c < 0x20 then reject_in_string c; + match c with + | '\\' -> + (* The escaped byte is still inside the literal: a raw + control byte hiding behind a backslash must not ride + through on the two-byte skip. *) + if !i + 1 < len && Char.code raw.[!i + 1] < 0x20 then + reject_in_string raw.[!i + 1]; + i := !i + 2 + | '"' -> + closed := true; + incr i + | _ -> incr i + done + end + else begin + if Char.code c < 0x20 && c <> '\t' && c <> '\n' && c <> '\r' then + raise + (Control_char_rejected + (Printf.sprintf + "unsupported raw control character (0x%02x) between tokens" + (Char.code c))); + incr i + end + done + +(** Raised when the raw input is not well-formed UTF-8 (RFC 3629). *) +exception Invalid_utf8 of string + +(* CROSS-LINEAGE CONTRACT: the raw input bytes must be well-formed + UTF-8 (RFC 3629) in every lineage library. yojson passes invalid + byte sequences through string literals verbatim (a stray 0x85 + continuation byte or a bare 0xE0 lead survives into the canonical + bytes), where C++/Rust/Haskell refuse — so validation must happen on + the raw bytes before parse. Sibling of reject_unsupported_numbers / + check_no_raw_control_chars, but no skip-strings walk: UTF-8 + well-formedness is a property of the whole byte stream, inside and + outside string literals alike. Rejects: stray continuation bytes, + truncated sequences, overlong encodings (0xC0/0xC1 leads, + 0xE0 0x80-0x9F, 0xF0 0x80-0x8F), encoded surrogates + (0xED 0xA0-0xBF), and code points above U+10FFFF (0xF4 0x90-0xBF + second bytes, 0xF5-0xFF leads). *) +let check_utf8 (raw : string) : unit = + let len = String.length raw in + let reject i what = + raise + (Invalid_utf8 + (Printf.sprintf "invalid UTF-8 (%s) at byte offset %d" what i)) + in + let cont i = + (* A required continuation byte: 0x80-0xBF, and present at all. *) + if i >= len then reject i "truncated sequence" + else + let b = Char.code raw.[i] in + if b < 0x80 || b > 0xBF then reject i "truncated sequence" + in + let i = ref 0 in + while !i < len do + let b0 = Char.code raw.[!i] in + if b0 < 0x80 then incr i + else if b0 < 0xC0 then reject !i "stray continuation byte" + else if b0 < 0xC2 then + (* 0xC0/0xC1 can only encode U+0000-U+007F in two bytes. *) + reject !i "overlong encoding" + else if b0 < 0xE0 then begin + cont (!i + 1); + i := !i + 2 + end + else if b0 < 0xF0 then begin + (if !i + 1 < len then + let b1 = Char.code raw.[!i + 1] in + if b0 = 0xE0 && b1 >= 0x80 && b1 <= 0x9F then + reject !i "overlong encoding" + else if b0 = 0xED && b1 >= 0xA0 && b1 <= 0xBF then + (* U+D800-U+DFFF: surrogate code points are not scalar values. *) + reject !i "encoded surrogate"); + cont (!i + 1); + cont (!i + 2); + i := !i + 3 + end + else if b0 < 0xF5 then begin + (if !i + 1 < len then + let b1 = Char.code raw.[!i + 1] in + if b0 = 0xF0 && b1 >= 0x80 && b1 <= 0x8F then + reject !i "overlong encoding" + else if b0 = 0xF4 && b1 >= 0x90 && b1 <= 0xBF then + reject !i "code point above U+10FFFF"); + cont (!i + 1); + cont (!i + 2); + cont (!i + 3); + i := !i + 4 + end + else + (* 0xF5-0xFF leads would encode code points above U+10FFFF. *) + reject !i "code point above U+10FFFF" + done + +(** Raised when the raw input carries a token outside strict RFC 8259: + an unquoted object key or other bare identifier (only true/false/null + are legal), or comment syntax. *) +exception Nonstandard_token of string + +(* CROSS-LINEAGE CONTRACT: strict RFC 8259 token grammar in every + lineage library. yojson's Safe parser is lax where C++/Rust/Haskell + refuse: it accepts unquoted object keys ({tz:true}), // and block + comments, and the NaN/Infinity/-Infinity literals. All of these are + LEXICAL properties — by parse time an unquoted key is an ordinary + string and Infinity is just a float — so this is a raw-text pass + using the same skip-strings walk as reject_unsupported_numbers. + Outside string literals, a letter may only begin the exact tokens + true / false / null, and '/' (comment syntax) never appears at all. + Number tokens are consumed opaquely so the 'e' in 1e2 is never + misread as an identifier (the number-domain pass owns that error). *) +let check_strict_tokens (raw : string) : unit = + let len = String.length raw in + let is_ident_char c = + (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c = '_' + || (c >= '0' && c <= '9') + in + let i = ref 0 in + while !i < len do + let c = raw.[!i] in + if c = '"' then begin + incr i; + let closed = ref false in + while (not !closed) && !i < len do + match raw.[!i] with + | '\\' -> i := !i + 2 + | '"' -> + closed := true; + incr i + | _ -> incr i + done + end + else if c = '-' || (c >= '0' && c <= '9') then + (* Consume a number token opaquely — same char set as + reject_unsupported_numbers, which owns number-domain errors. *) + while + !i < len + && + match raw.[!i] with + | '0' .. '9' | '-' | '+' | '.' | 'e' | 'E' -> true + | _ -> false + do + incr i + done + else if c = '/' then + raise (Nonstandard_token "unsupported comment syntax") + else if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c = '_' then begin + let start = !i in + while !i < len && is_ident_char raw.[!i] do + incr i + done; + let tok = String.sub raw start (!i - start) in + if tok <> "true" && tok <> "false" && tok <> "null" then + raise + (Nonstandard_token + ("unsupported bare token (unquoted key or non-JSON literal): " + ^ tok)) + end + else incr i + done + (* CROSS-LINEAGE CONTRACT: objects with duplicate member names (at any depth) must be rejected by all seven lineage libraries. yojson's Assoc is a plain pair list that preserves EVERY member — including diff --git a/ocaml/test/conformance_test.ml b/ocaml/test/conformance_test.ml index e7aa660..e5c8c10 100644 --- a/ocaml/test/conformance_test.ml +++ b/ocaml/test/conformance_test.ml @@ -246,6 +246,141 @@ let test_literal_backslash_udc00_allowed () = even run pairs off into literal backslashes and must pass. *) Canonical_json.check_no_lone_surrogates {|{"s":"\\udc00"}|} +(* Raw-control-byte enforcement is a LEXICAL pass over the raw text — + yojson accepts an unescaped 0x01-0x1F byte inside a string literal, + indistinguishable from the escaped spelling by parse time. The + payloads below use OCaml string escapes ("\t", "\x1e") in REGULAR + (non-{|...|}) strings, so the compiler emits the raw control byte + into the test input — never a literal control byte in this source + file. *) +let test_raw_control_in_string_rejected () = + Alcotest.check_raises "raw TAB inside string literal rejected" + (Canonical_json.Control_char_rejected + "unsupported raw control character (0x09) in string literal") + (fun () -> Canonical_json.check_no_raw_control_chars "{\"s\":\"a\tb\"}"); + Alcotest.check_raises "raw 0x1e inside string literal rejected" + (Canonical_json.Control_char_rejected + "unsupported raw control character (0x1e) in string literal") + (fun () -> Canonical_json.check_no_raw_control_chars "{\"s\":\"a\x1eb\"}"); + (* A raw control byte immediately after a backslash is still inside + the literal and must not ride through on the escape's 2-byte skip. *) + Alcotest.check_raises "raw 0x01 behind a backslash rejected" + (Canonical_json.Control_char_rejected + "unsupported raw control character (0x01) in string literal") + (fun () -> + Canonical_json.check_no_raw_control_chars "{\"s\":\"a\\\x01b\"}") + +let test_raw_control_between_tokens_rejected () = + Alcotest.check_raises "raw 0x02 between tokens rejected" + (Canonical_json.Control_char_rejected + "unsupported raw control character (0x02) between tokens") + (fun () -> + Canonical_json.check_no_raw_control_chars "{\"a\":1,\x02\"b\":2}"); + (* Form feed is a control byte, NOT legal JSON whitespace. *) + Alcotest.check_raises "raw form feed between tokens rejected" + (Canonical_json.Control_char_rejected + "unsupported raw control character (0x0c) between tokens") + (fun () -> Canonical_json.check_no_raw_control_chars "{\"a\":\x0c1}") + +let test_escaped_controls_and_legal_ws_allowed () = + (* Escaped forms stay ACCEPTED: backslash-t / backslash-u001f are + printable bytes in the raw text (the {|...|} raw-string literals + below contain a real backslash, no control byte). *) + Canonical_json.check_no_raw_control_chars {|{"s":"a\tb"}|}; + Canonical_json.check_no_raw_control_chars "{\"s\":\"a\\u001fb\"}"; + (* Legal insignificant whitespace between tokens: TAB/LF/CR/space. *) + Canonical_json.check_no_raw_control_chars "{\"a\":\t1}"; + Canonical_json.check_no_raw_control_chars "\n{\"a\": 1}\r\n" + +(* UTF-8 well-formedness is a check on the RAW bytes — yojson passes + invalid sequences through string literals verbatim. Payloads below + use OCaml "\xNN" escapes in regular strings so the compiler emits + the raw bytes; this source file never contains a raw invalid byte. *) +let test_invalid_utf8_rejected () = + let reject label input msg = + Alcotest.check_raises label (Canonical_json.Invalid_utf8 msg) (fun () -> + Canonical_json.check_utf8 input) + in + reject "stray continuation byte rejected" "\"a\x85b\"" + "invalid UTF-8 (stray continuation byte) at byte offset 2"; + reject "bare lead byte before ASCII rejected" "{\"\xe0a\":[]}" + "invalid UTF-8 (truncated sequence) at byte offset 3"; + reject "truncated 2-byte sequence at end rejected" "\"\xc3\"" + "invalid UTF-8 (truncated sequence) at byte offset 2"; + reject "overlong 2-byte encoding (0xC0) rejected" "\"\xc0\xaf\"" + "invalid UTF-8 (overlong encoding) at byte offset 1"; + reject "overlong 2-byte encoding (0xC1) rejected" "\"\xc1\x81\"" + "invalid UTF-8 (overlong encoding) at byte offset 1"; + reject "overlong 3-byte encoding (0xE0 0x80) rejected" "\"\xe0\x80\x80\"" + "invalid UTF-8 (overlong encoding) at byte offset 1"; + reject "encoded surrogate (0xED 0xA0) rejected" "\"\xed\xa0\x80\"" + "invalid UTF-8 (encoded surrogate) at byte offset 1"; + reject "overlong 4-byte encoding (0xF0 0x80) rejected" "\"\xf0\x80\x80\x80\"" + "invalid UTF-8 (overlong encoding) at byte offset 1"; + reject "above U+10FFFF (0xF4 0x90) rejected" "\"\xf4\x90\x80\x80\"" + "invalid UTF-8 (code point above U+10FFFF) at byte offset 1"; + reject "0xF5 lead byte rejected" "\"\xf5\x80\x80\x80\"" + "invalid UTF-8 (code point above U+10FFFF) at byte offset 1"; + reject "0xFF lead byte rejected" "\"\xff\"" + "invalid UTF-8 (code point above U+10FFFF) at byte offset 1" + +let test_valid_utf8_allowed () = + (* Well-formed multi-byte sequences must pass: 2-byte (U+00E9), + 3-byte (U+20AC), 4-byte (U+1F600), and the boundary code points + U+E000 (first post-surrogate) and U+10FFFF (last scalar value). *) + Canonical_json.check_utf8 "{\"z\":1,\"a\":\"\xc3\xa9\"}"; + Canonical_json.check_utf8 "{\"x\":\"\xe2\x82\xac\"}"; + Canonical_json.check_utf8 "{\"x\":\"\xf0\x9f\x98\x80\"}"; + Canonical_json.check_utf8 "\"\xee\x80\x80\""; + Canonical_json.check_utf8 "\"\xf4\x8f\xbf\xbf\""; + Canonical_json.check_utf8 "\"plain ascii\"" + +(* Strict token grammar is a LEXICAL pass — yojson's Safe parser + accepts unquoted object keys, // and block comments, and the + NaN/Infinity literals, none of which survive to the parsed value. *) +let test_unquoted_key_rejected () = + Alcotest.check_raises "unquoted object key rejected" + (Canonical_json.Nonstandard_token + "unsupported bare token (unquoted key or non-JSON literal): tz") + (fun () -> Canonical_json.check_strict_tokens {|{tz:true}|}) + +let test_nonstandard_literals_rejected () = + let reject label input tok = + Alcotest.check_raises label + (Canonical_json.Nonstandard_token + ("unsupported bare token (unquoted key or non-JSON literal): " ^ tok)) + (fun () -> Canonical_json.check_strict_tokens input) + in + reject "NaN literal rejected" {|{"x":NaN}|} "NaN"; + reject "Infinity literal rejected" {|{"x":Infinity}|} "Infinity"; + (* The '-' is consumed as an (empty-domain) number token; the + identifier that follows is what trips the scanner. *) + reject "-Infinity literal rejected" {|{"x":-Infinity}|} "Infinity"; + reject "capitalized True rejected" {|True|} "True" + +let test_comment_syntax_rejected () = + Alcotest.check_raises "line comment rejected" + (Canonical_json.Nonstandard_token "unsupported comment syntax") + (fun () -> Canonical_json.check_strict_tokens "{\"a\":1 // c\n}"); + Alcotest.check_raises "block comment rejected" + (Canonical_json.Nonstandard_token "unsupported comment syntax") + (fun () -> Canonical_json.check_strict_tokens {|{"a":/* c */1}|}) + +let test_strict_tokens_legal_input_allowed () = + (* The three legal bare literals pass, standalone and as values. *) + Canonical_json.check_strict_tokens "true"; + Canonical_json.check_strict_tokens "false"; + Canonical_json.check_strict_tokens "null"; + Canonical_json.check_strict_tokens {|{"a":true,"b":false,"c":null}|}; + (* Letters, slashes, and NaN/Infinity spellings INSIDE string + literals are ordinary string content and must pass. *) + Canonical_json.check_strict_tokens {|{"s":"NaN and Infinity and // x"}|}; + Canonical_json.check_strict_tokens {|{"url":"http://example.com/a"}|}; + (* Number tokens are consumed opaquely: the 'e' in 1e2 must never be + misread as a bare identifier (the number pass owns that error). *) + Canonical_json.check_strict_tokens {|{"x":1e2}|}; + Canonical_json.check_strict_tokens {|{"x":-1.5,"y":100}|} + (* Shortest-roundtrip float formatting: 0.1 must canonicalize as "0.1", not the %.17g spelling "0.10000000000000001" (RFC 8785 §3.2.2.3 / ECMA-262 §7.1.12.1). *) @@ -330,6 +465,33 @@ let () = Alcotest.test_case "literal backslash + udc00 text allowed" `Quick test_literal_backslash_udc00_allowed; ] ); + ( "raw_control_char_rejection", + [ + Alcotest.test_case "raw control byte inside string rejected" `Quick + test_raw_control_in_string_rejected; + Alcotest.test_case "raw control byte between tokens rejected" `Quick + test_raw_control_between_tokens_rejected; + Alcotest.test_case "escaped controls and legal whitespace allowed" + `Quick test_escaped_controls_and_legal_ws_allowed; + ] ); + ( "utf8_validation", + [ + Alcotest.test_case "invalid UTF-8 byte sequences rejected" `Quick + test_invalid_utf8_rejected; + Alcotest.test_case "well-formed UTF-8 allowed" `Quick + test_valid_utf8_allowed; + ] ); + ( "strict_token_grammar", + [ + Alcotest.test_case "unquoted object key rejected" `Quick + test_unquoted_key_rejected; + Alcotest.test_case "NaN/Infinity/True literals rejected" `Quick + test_nonstandard_literals_rejected; + Alcotest.test_case "comment syntax rejected" `Quick + test_comment_syntax_rejected; + Alcotest.test_case "legal literals and string content allowed" + `Quick test_strict_tokens_legal_input_allowed; + ] ); ( "float_formatting", [ Alcotest.test_case "shortest-roundtrip float formatting" `Quick