From 84bb0956c09ce3183e707fcc64e145157b489593 Mon Sep 17 00:00:00 2001 From: MDA2AV Date: Sun, 7 Jun 2026 16:01:44 +0000 Subject: [PATCH] Harden json related tests --- scripts/validate.sh | 133 ++++++++++++------ .../docs/running-locally/scripts/validate.md | 2 +- .../gateway/gateway-h2/validation.md | 2 +- .../gateway/gateway-h3/validation.md | 2 +- .../gateway/production-stack/validation.md | 2 +- .../h1/isolated/json-compressed/validation.md | 4 +- .../h1/isolated/json-processing/validation.md | 4 +- .../h1/isolated/json-tls/validation.md | 2 +- .../test-profiles/h2/json-h2c/validation.md | 2 +- 9 files changed, 100 insertions(+), 53 deletions(-) diff --git a/scripts/validate.sh b/scripts/validate.sh index c04c095ab..f3d336421 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -589,28 +589,34 @@ m = $jm d = json.load(sys.stdin) count = d.get('count', 0) items = d.get('items', []) -has_total = all('total' in item for item in items) if items else False +def valid_item(it): + r = it.get('rating') + return ('id' in it and 'name' in it and 'category' in it and 'price' in it + and 'quantity' in it and 'total' in it + and isinstance(it.get('tags'), list) and isinstance(it.get('active'), bool) + and isinstance(r, dict) and 'score' in r and 'count' in r) +valid = all(valid_item(it) for it in items) if items else False correct_totals = True for item in items: - expected = item['price'] * item['quantity'] * m + expected = item.get('price', 0) * item.get('quantity', 0) * m if item.get('total', 0) != expected: correct_totals = False break -print(f'{count} {has_total} {correct_totals}') +print(f'{count} {valid} {correct_totals}') " 2>/dev/null || echo "0 False False") json_count=$(echo "$json_result" | cut -d' ' -f1) - json_total=$(echo "$json_result" | cut -d' ' -f2) + json_valid=$(echo "$json_result" | cut -d' ' -f2) json_correct=$(echo "$json_result" | cut -d' ' -f3) - if [ "$json_count" = "$jcount" ] && [ "$json_total" = "True" ] && [ "$json_correct" = "True" ]; then + if [ "$json_count" = "$jcount" ] && [ "$json_valid" = "True" ] && [ "$json_correct" = "True" ]; then : else - fail_with_link "[GET /json/$jcount?m=$jm]: count=$json_count, has_total=$json_total, correct_totals=$json_correct" "$JSON_DOCS" + fail_with_link "[GET /json/$jcount?m=$jm]: count=$json_count, schema=$json_valid, correct_totals=$json_correct" "$JSON_DOCS" json_fail=true fi done if [ "$json_fail" = "false" ]; then - echo " PASS [GET /json/{count}?m=X] (4 counts with multipliers verified)" + echo " PASS [GET /json/{count}?m=X] (4 counts × multipliers + full item schema verified)" PASS=$((PASS + 1)) fi @@ -648,28 +654,34 @@ m = $jcm d = json.load(sys.stdin) count = d.get('count', 0) items = d.get('items', []) -has_total = all('total' in item for item in items) if items else False +def valid_item(it): + r = it.get('rating') + return ('id' in it and 'name' in it and 'category' in it and 'price' in it + and 'quantity' in it and 'total' in it + and isinstance(it.get('tags'), list) and isinstance(it.get('active'), bool) + and isinstance(r, dict) and 'score' in r and 'count' in r) +valid = all(valid_item(it) for it in items) if items else False correct_totals = True for item in items: - expected = item['price'] * item['quantity'] * m + expected = item.get('price', 0) * item.get('quantity', 0) * m if item.get('total', 0) != expected: correct_totals = False break -print(f'{count} {has_total} {correct_totals}') +print(f'{count} {valid} {correct_totals}') " 2>/dev/null || echo "0 False False") jc_count=$(echo "$jc_result" | cut -d' ' -f1) - jc_total=$(echo "$jc_result" | cut -d' ' -f2) + jc_valid=$(echo "$jc_result" | cut -d' ' -f2) jc_correct=$(echo "$jc_result" | cut -d' ' -f3) - if [ "$jc_count" = "$jccount" ] && [ "$jc_total" = "True" ] && [ "$jc_correct" = "True" ]; then + if [ "$jc_count" = "$jccount" ] && [ "$jc_valid" = "True" ] && [ "$jc_correct" = "True" ]; then : else - fail_with_link "[json-comp /json/$jccount?m=$jcm]: count=$jc_count, has_total=$jc_total, correct=$jc_correct" "$JSONCOMP_DOCS" + fail_with_link "[json-comp /json/$jccount?m=$jcm]: count=$jc_count, schema=$jc_valid, correct=$jc_correct" "$JSONCOMP_DOCS" jc_fail=true fi done if [ "$jc_fail" = "false" ]; then - echo " PASS [json-comp response] (3 counts with multipliers, compressed)" + echo " PASS [json-comp response] (3 counts × multipliers, compressed, full item schema)" PASS=$((PASS + 1)) fi @@ -711,28 +723,34 @@ m = $jtm d = json.load(sys.stdin) count = d.get('count', 0) items = d.get('items', []) -has_total = all('total' in item for item in items) if items else False +def valid_item(it): + r = it.get('rating') + return ('id' in it and 'name' in it and 'category' in it and 'price' in it + and 'quantity' in it and 'total' in it + and isinstance(it.get('tags'), list) and isinstance(it.get('active'), bool) + and isinstance(r, dict) and 'score' in r and 'count' in r) +valid = all(valid_item(it) for it in items) if items else False correct_totals = True for item in items: - expected = item['price'] * item['quantity'] * m + expected = item.get('price', 0) * item.get('quantity', 0) * m if item.get('total', 0) != expected: correct_totals = False break -print(f'{count} {has_total} {correct_totals}') +print(f'{count} {valid} {correct_totals}') " 2>/dev/null || echo "0 False False") jt_count=$(echo "$jt_result" | cut -d' ' -f1) - jt_total=$(echo "$jt_result" | cut -d' ' -f2) + jt_valid=$(echo "$jt_result" | cut -d' ' -f2) jt_correct=$(echo "$jt_result" | cut -d' ' -f3) - if [ "$jt_count" = "$jtcount" ] && [ "$jt_total" = "True" ] && [ "$jt_correct" = "True" ]; then + if [ "$jt_count" = "$jtcount" ] && [ "$jt_valid" = "True" ] && [ "$jt_correct" = "True" ]; then : else - fail_with_link "[json-tls /json/$jtcount?m=$jtm]: count=$jt_count, has_total=$jt_total, correct=$jt_correct" "$JSONTLS_DOCS" + fail_with_link "[json-tls /json/$jtcount?m=$jtm]: count=$jt_count, schema=$jt_valid, correct=$jt_correct" "$JSONTLS_DOCS" jt_fail=true fi done if [ "$jt_fail" = "false" ]; then - echo " PASS [json-tls response] (3 (count, m) pairs over TLS)" + echo " PASS [json-tls response] (3 (count, m) pairs over TLS, full item schema)" PASS=$((PASS + 1)) fi @@ -909,22 +927,39 @@ if has_test "json-h2c"; then "http://localhost:$H2C_PORT/json/$jcount?m=$jm" 2>/dev/null || true) parsed=$(echo "$resp" | python3 -c " import sys, json +m = $jm d = json.load(sys.stdin) count = d.get('count', -1) -items_n = len(d.get('items', [])) -print(f'{count} {items_n}') -" 2>/dev/null || echo "-1 -1") +items = d.get('items', []) +items_n = len(items) +def valid_item(it): + r = it.get('rating') + return ('id' in it and 'name' in it and 'category' in it and 'price' in it + and 'quantity' in it and 'total' in it + and isinstance(it.get('tags'), list) and isinstance(it.get('active'), bool) + and isinstance(r, dict) and 'score' in r and 'count' in r) +valid = all(valid_item(it) for it in items) if items else False +correct_totals = True +for item in items: + expected = item.get('price', 0) * item.get('quantity', 0) * m + if item.get('total', 0) != expected: + correct_totals = False + break +print(f'{count} {items_n} {valid} {correct_totals}') +" 2>/dev/null || echo "-1 -1 False False") pc=$(echo "$parsed" | cut -d' ' -f1) pn=$(echo "$parsed" | cut -d' ' -f2) - if [ "$pc" = "$jcount" ] && [ "$pn" = "$jcount" ]; then + pv=$(echo "$parsed" | cut -d' ' -f3) + ptot=$(echo "$parsed" | cut -d' ' -f4) + if [ "$pc" = "$jcount" ] && [ "$pn" = "$jcount" ] && [ "$pv" = "True" ] && [ "$ptot" = "True" ]; then : else - fail_with_link "[GET /json/$jcount?m=$jm (h2c)]: count=$pc, items=$pn, expected $jcount" "$JSON_H2C_DOCS" + fail_with_link "[GET /json/$jcount?m=$jm (h2c)]: count=$pc, items=$pn, schema=$pv, correct_totals=$ptot, expected $jcount" "$JSON_H2C_DOCS" json_h2c_fail=true fi done if [ "$json_h2c_fail" = "false" ]; then - echo " PASS [GET /json/{count}?m=X over h2c] (4 counts verified)" + echo " PASS [GET /json/{count}?m=X over h2c] (4 counts × multipliers, full item schema + totals)" PASS=$((PASS + 1)) fi fi @@ -1329,31 +1364,37 @@ _validate_gateway() { -sk --http2 "https://localhost:$GW_PORT/static/nonexistent.txt" # 5. JSON endpoint — valid JSON with computed totals - local gw_json_response gw_json_result gw_json_count gw_json_total gw_json_correct + local gw_json_response gw_json_result gw_json_count gw_json_valid gw_json_correct gw_json_response=$(curl -sk --max-time 30 --http2 "https://localhost:$GW_PORT/json/50" || true) gw_json_result=$(echo "$gw_json_response" | python3 -c " import sys, json d = json.load(sys.stdin) count = d.get('count', 0) items = d.get('items', []) -has_total = all('total' in item for item in items) if items else False +def valid_item(it): + r = it.get('rating') + return ('id' in it and 'name' in it and 'category' in it and 'price' in it + and 'quantity' in it and 'total' in it + and isinstance(it.get('tags'), list) and isinstance(it.get('active'), bool) + and isinstance(r, dict) and 'score' in r and 'count' in r) +valid = all(valid_item(it) for it in items) if items else False correct_totals = True for item in items: - expected = round(item['price'] * item['quantity'], 2) + expected = round(item.get('price', 0) * item.get('quantity', 0), 2) if abs(item.get('total', 0) - expected) > 0.02: correct_totals = False break -print(f'{count} {has_total} {correct_totals}') +print(f'{count} {valid} {correct_totals}') " 2>/dev/null || echo "0 False False") gw_json_count=$(echo "$gw_json_result" | cut -d' ' -f1) - gw_json_total=$(echo "$gw_json_result" | cut -d' ' -f2) + gw_json_valid=$(echo "$gw_json_result" | cut -d' ' -f2) gw_json_correct=$(echo "$gw_json_result" | cut -d' ' -f3) - if [ "$gw_json_count" = "50" ] && [ "$gw_json_total" = "True" ] && [ "$gw_json_correct" = "True" ]; then - echo " PASS [gateway /json] (50 items, totals correct)" + if [ "$gw_json_count" = "50" ] && [ "$gw_json_valid" = "True" ] && [ "$gw_json_correct" = "True" ]; then + echo " PASS [gateway /json] (50 items, full schema, totals correct)" PASS=$((PASS + 1)) else - fail_with_link "[gateway /json]: count=$gw_json_count, has_total=$gw_json_total, correct=$gw_json_correct" "$gateway_docs" + fail_with_link "[gateway /json]: count=$gw_json_count, schema=$gw_json_valid, correct=$gw_json_correct" "$gateway_docs" fi check_header "gateway /json Content-Type" "Content-Type" "application/json" "$gateway_docs" \ @@ -1508,31 +1549,37 @@ _validate_production_stack() { -sk --http2 "https://localhost:$GW_PORT/public/baseline?a=$GW_A&b=$GW_B" # 4. Public JSON — no auth, no cache, returns count items with totals - local gw_json_response gw_json_result gw_json_count gw_json_total gw_json_correct + local gw_json_response gw_json_result gw_json_count gw_json_valid gw_json_correct gw_json_response=$(curl -sk --max-time 30 --http2 "https://localhost:$GW_PORT/public/json/25" || true) gw_json_result=$(echo "$gw_json_response" | python3 -c " import sys, json d = json.load(sys.stdin) count = d.get('count', 0) items = d.get('items', []) -has_total = all('total' in item for item in items) if items else False +def valid_item(it): + r = it.get('rating') + return ('id' in it and 'name' in it and 'category' in it and 'price' in it + and 'quantity' in it and 'total' in it + and isinstance(it.get('tags'), list) and isinstance(it.get('active'), bool) + and isinstance(r, dict) and 'score' in r and 'count' in r) +valid = all(valid_item(it) for it in items) if items else False correct_totals = True for item in items: - expected = round(item['price'] * item['quantity'], 2) + expected = round(item.get('price', 0) * item.get('quantity', 0), 2) if abs(item.get('total', 0) - expected) > 0.02: correct_totals = False break -print(f'{count} {has_total} {correct_totals}') +print(f'{count} {valid} {correct_totals}') " 2>/dev/null || echo "0 False False") gw_json_count=$(echo "$gw_json_result" | cut -d' ' -f1) - gw_json_total=$(echo "$gw_json_result" | cut -d' ' -f2) + gw_json_valid=$(echo "$gw_json_result" | cut -d' ' -f2) gw_json_correct=$(echo "$gw_json_result" | cut -d' ' -f3) - if [ "$gw_json_count" = "25" ] && [ "$gw_json_total" = "True" ] && [ "$gw_json_correct" = "True" ]; then - echo " PASS [$profile /public/json/25] (25 items, totals correct)" + if [ "$gw_json_count" = "25" ] && [ "$gw_json_valid" = "True" ] && [ "$gw_json_correct" = "True" ]; then + echo " PASS [$profile /public/json/25] (25 items, full schema, totals correct)" PASS=$((PASS + 1)) else - fail_with_link "[$profile /public/json/25]: count=$gw_json_count, has_total=$gw_json_total, correct=$gw_json_correct" "$docs_url" + fail_with_link "[$profile /public/json/25]: count=$gw_json_count, schema=$gw_json_valid, correct=$gw_json_correct" "$docs_url" fi # 5. Auth wall (GET) — /api/* without a cookie must return 401 diff --git a/site/content/docs/running-locally/scripts/validate.md b/site/content/docs/running-locally/scripts/validate.md index 9f85e4366..432ef35dd 100644 --- a/site/content/docs/running-locally/scripts/validate.md +++ b/site/content/docs/running-locally/scripts/validate.md @@ -45,7 +45,7 @@ Several validations use randomized inputs to detect hardcoded responses: PASS [POST /baseline11?a=13&b=42 body=20] ... [test] json endpoint - PASS [GET /json] (50 items, totals computed correctly) + PASS [GET /json/{count}?m=X] (4 counts × multipliers + full item schema verified) PASS [GET /json Content-Type] (Content-Type: application/json) ... diff --git a/site/content/docs/test-profiles/gateway/gateway-h2/validation.md b/site/content/docs/test-profiles/gateway/gateway-h2/validation.md index b9410c12c..a8fb62fc6 100644 --- a/site/content/docs/test-profiles/gateway/gateway-h2/validation.md +++ b/site/content/docs/test-profiles/gateway/gateway-h2/validation.md @@ -44,7 +44,7 @@ Sends `GET /static/nonexistent.txt` over HTTP/2 and verifies the response is **H Sends `GET /json` over HTTP/2 and validates: - Response contains exactly **50 items** -- Every item has a `total` field +- Every item carries the full schema — `id`, `name`, `category`, `price`, `quantity`, `active`, `tags` (array), `rating` (object with `score` and `count`), and `total` - Each `total` is correctly computed as `price * quantity` (rounded to 2 decimal places) This is the same validation as the [JSON Processing test](../../h1/isolated/json-processing/validation), but routed through the proxy. diff --git a/site/content/docs/test-profiles/gateway/gateway-h3/validation.md b/site/content/docs/test-profiles/gateway/gateway-h3/validation.md index ce7907435..6bae17289 100644 --- a/site/content/docs/test-profiles/gateway/gateway-h3/validation.md +++ b/site/content/docs/test-profiles/gateway/gateway-h3/validation.md @@ -19,7 +19,7 @@ Same set as [Gateway-64 validation](../gateway-h2/validation/): 4. `/static/app.js` — `Content-Type: application/javascript` 5. `/static/app.js` — non-zero response body 6. `/static/nonexistent.txt` — HTTP 404 -7. `/json/50` — returns 50 items with computed `total` field per item +7. `/json/50` — returns 50 items, each with the full schema (`id`, `name`, `category`, `price`, `quantity`, `active`, `tags`, `rating` with `score`+`count`) and a computed `total` 8. `/json/50` — `Content-Type: application/json` 9. `/async-db?min=10&max=50&limit=50` — returns 1–50 items with nested `rating`, `tags`, boolean `active` 10. `/async-db` — `Content-Type: application/json` diff --git a/site/content/docs/test-profiles/gateway/production-stack/validation.md b/site/content/docs/test-profiles/gateway/production-stack/validation.md index 15bc739fc..b4340e495 100644 --- a/site/content/docs/test-profiles/gateway/production-stack/validation.md +++ b/site/content/docs/test-profiles/gateway/production-stack/validation.md @@ -15,7 +15,7 @@ weight: 2 ### Public endpoints (no auth) 4. **`/public/baseline?a=13&b=42`** — returns `55` 5. **`/public/baseline?a=&b=`** — correct sum (anti-cheat) -6. **`/public/json/25`** — 25 items with computed `total = price × quantity` +6. **`/public/json/25`** — 25 items, each with the full schema (`id`, `name`, `category`, `price`, `quantity`, `active`, `tags`, `rating` with `score`+`count`) and computed `total = price × quantity` ### Auth wall — JWT required 7. **`GET /api/items/1`** (no Authorization header) — **HTTP 401** diff --git a/site/content/docs/test-profiles/h1/isolated/json-compressed/validation.md b/site/content/docs/test-profiles/h1/isolated/json-compressed/validation.md index c8e7cb2ef..c72e415bd 100644 --- a/site/content/docs/test-profiles/h1/isolated/json-compressed/validation.md +++ b/site/content/docs/test-profiles/h1/isolated/json-compressed/validation.md @@ -28,10 +28,10 @@ Three requests are sent with different counts and multipliers: For each response, after decompressing, the validator checks: 1. `count` field equals the route count -2. Every item in `items` has a `total` field +2. Every item in `items` contains the full schema — `id`, `name`, `category`, `price`, `quantity`, `active`, `tags` (array), `rating` (object with `score` and `count`), and `total` 3. `total == price * quantity * m` for every item (integer, exact) -Any missing field or incorrect arithmetic is a failure. This confirms the server honors the `m` parameter and applies it per item. +Any missing field or incorrect arithmetic is a failure. Partial payloads that omit fields are rejected. This confirms the server honors the `m` parameter and applies it per item. ### No Content-Encoding when Accept-Encoding is absent diff --git a/site/content/docs/test-profiles/h1/isolated/json-processing/validation.md b/site/content/docs/test-profiles/h1/isolated/json-processing/validation.md index 638e1d74d..cfb6c2b07 100644 --- a/site/content/docs/test-profiles/h1/isolated/json-processing/validation.md +++ b/site/content/docs/test-profiles/h1/isolated/json-processing/validation.md @@ -9,8 +9,8 @@ The following checks are executed by `validate.sh` for every framework subscribe Sends `GET /json/{count}` for counts **12, 22, 31, and 50** (different from the benchmark counts to prevent hardcoded responses). For each request, verifies: - The response contains exactly **count** items -- Every item has a `total` field -- Each `total` is correctly computed as `price * quantity`, rounded to 2 decimal places (tolerance: 0.01) +- Every item contains the full schema — `id`, `name`, `category`, `price`, `quantity`, `active`, `tags` (array), `rating` (object with `score` and `count`), and `total`. Partial payloads that omit fields are rejected. +- Each `total` is correctly computed as `price * quantity * m` ## Content-Type header diff --git a/site/content/docs/test-profiles/h1/isolated/json-tls/validation.md b/site/content/docs/test-profiles/h1/isolated/json-tls/validation.md index dd957dd24..aab5338b5 100644 --- a/site/content/docs/test-profiles/h1/isolated/json-tls/validation.md +++ b/site/content/docs/test-profiles/h1/isolated/json-tls/validation.md @@ -27,7 +27,7 @@ Three requests are sent over HTTPS on port 8081 with different counts and multip For each response the validator checks: 1. `count` field equals the route count -2. Every item in `items` has a `total` field +2. Every item in `items` contains the full schema — `id`, `name`, `category`, `price`, `quantity`, `active`, `tags` (array), `rating` (object with `score` and `count`), and `total` 3. `total == price * quantity * m` for every item (integer, exact) These `(count, m)` pairs are deliberately **different** from the `json-comp` validation pairs so a framework that tries to cache validation results across profiles can't pass both. diff --git a/site/content/docs/test-profiles/h2/json-h2c/validation.md b/site/content/docs/test-profiles/h2/json-h2c/validation.md index 1e0182796..000f38016 100644 --- a/site/content/docs/test-profiles/h2/json-h2c/validation.md +++ b/site/content/docs/test-profiles/h2/json-h2c/validation.md @@ -14,4 +14,4 @@ Response must include `Content-Type: application/json` (charset suffix permitted ## Correctness across four (count, m) pairs -Sends four requests with `(count, m)` ∈ `{(12, 3), (22, 7), (31, 2), (50, 5)}` — deliberately distinct from the benchmark's seven rotation pairs so any caching-by-key strategy returns stale data. Each response's `count` field must equal the requested count, and `items.length` must equal the count. +Sends four requests with `(count, m)` ∈ `{(12, 3), (22, 7), (31, 2), (50, 5)}` — deliberately distinct from the benchmark's seven rotation pairs so any caching-by-key strategy returns stale data. For each response: the `count` field must equal the requested count, `items.length` must equal the count, every item must carry the full schema (`id`, `name`, `category`, `price`, `quantity`, `active`, `tags` array, `rating` object with `score`+`count`, and `total`), and `total == price * quantity * m`. Partial payloads that omit fields are rejected.