From e4285559c3d723dc83c994337812b8130d68c522 Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 05:49:07 -0400 Subject: [PATCH 1/5] tables: a *string blob on a message body carries no content rule, red first (#632) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/SPEC-TABLES.md §3.1 states kinds 12 and 33's content rule MET AT A NODE: "A TEXT blob's CONTENT is refused on the same terms", so a *string blob whose bytes are not well-formed UTF-8, or which carries a zero byte, is damage. §3.3 says a form-2 body's content rules are §3's, unchanged in what they reject, and that what differs is only the recovery, which a bit stream does not have. Neither engine carries it at the message site, where both carry it at the file site (decodenodes.go and pointers.go). The two gates are the page's rows over blobdemo's Catalog, whose numbering reaches a *string blob through note and a *bytes blob through thumb. test/tables/message_blob_main.cpp is the C++ reference's and TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule is the oracle's. Both are RED at this commit: the truncated sequence, the zero byte, the overlong encoding and the lead byte 0xFF all load with a silent report. The last row of each is a *bytes blob carrying the same bytes, which must stay silent: a *bytes blob is bytes and never text, so the rule is the string id's alone. Co-Authored-By: Claude Opus 5 --- test/conformance/harness/messagerules_test.go | 78 +++++++ test/tables/message_blob_main.cpp | 190 ++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 test/tables/message_blob_main.cpp diff --git a/test/conformance/harness/messagerules_test.go b/test/conformance/harness/messagerules_test.go index b1dd2c4f5..2eddd5780 100644 --- a/test/conformance/harness/messagerules_test.go +++ b/test/conformance/harness/messagerules_test.go @@ -1512,3 +1512,81 @@ func TestTheMessageFormsTextContentRuleAndClamp(t *testing.T) { t.Errorf("a payload at the bound lands whole: kept %d bytes, %q", len(got), got) } } + +// TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule: kinds `12` and +// `33`'s content rule MET AT A NODE (docs/SPEC-TABLES.md §3.1) over a form-`2` +// body (§3.3). §3.1 refuses a text blob's CONTENT on the file form's own +// terms, and §3.3 says a form-`2` body's content rules are §3's, unchanged in +// what they reject: a `*string` blob whose bytes are not well-formed UTF-8, or +// which carries a zero byte, is DAMAGE and not data. What differs is only the +// recovery, which a bit stream does not have, so the damage is TERMINAL for +// the batch. Red if the message path places a blob the file path refuses. +// +// The instrument is `blobdemo`'s `Catalog`, whose numbering reaches a +// `*string` blob through `note` and a `*bytes` blob through `thumb`. A blob +// record is its own framing wherever it appears: the type reference, a length +// at thirty-two raw bits, the ALIGN, then the bytes verbatim. A record is +// numbered and placed whether or not a slot names it, so the rule is reached +// by the record alone. +func TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule(t *testing.T) { + _, _, u := corpus(t) + unit, err := u.get("blobdemo") + if err != nil { + t.Fatal(err) + } + model := tabletext.NewModel(unit) + v := vocabularyOf(t, unit) + // THE RESERVED IDS RIDE IN THE ANNOUNCEMENT'S TAIL whether or not a root + // names them (§3.1, §3.3), each a kind-0 entry that frames nothing + node := slotOf(t, v, ir.TableNodeWireId, 0) + stringType := slotOf(t, v, ir.StringWireTypeId, 0) + bytesType := slotOf(t, v, ir.BytesWireTypeId, 0) + body := func(typeSlot uint64, data []byte) []byte { + w := &bitw{} + w.put(node, v.RefBits()) + w.put(1, 32) + w.put(typeSlot, v.RefBits()) + w.put(uint64(len(data)), 32) + w.align() + w.bytes(data) + w.put(0, v.RefBits()) + return batchOf(w) + } + + // A WELL-FORMED BLOB IS ORDINARY, and the row is here first so a gate that + // went red by refusing everything is not one that holds the rule + _, ok, report, err := decodeOne(t, model, "Catalog", v, body(stringType, []byte("abcdefgh"))) + if err != nil || !ok || !report.Silent() { + t.Fatalf("a well-formed *string blob on a message body loads silently: ok=%v err=%v report=%+v", ok, err, report) + } + + // ILL-FORMED CONTENT IS DAMAGE AND IT IS TERMINAL: a truncated sequence, a + // zero byte, an overlong encoding and a lead byte UTF-8 never spells are + // each a payload that is not text at whatever length it arrived at + damaged := []struct { + name string + data []byte + }{ + {"a truncated sequence", []byte{'p', 'a', 'c', 'k', 0xC3}}, + {"a zero byte among the bytes", []byte{'p', 'a', 0x00, 'c', 'k'}}, + {"an overlong encoding", []byte{'p', 0xC0, 0x80, 'k'}}, + {"a lead byte UTF-8 never spells", []byte{'a', 'b', 'c', 0xFF, 'e', 'f', 'g', 'h'}}, + } + for _, dr := range damaged { + _, loaded, rep, derr := decodeOne(t, model, "Catalog", v, body(stringType, dr.data)) + if derr != nil { + t.Fatalf("%s: the decode errored: %v", dr.name, derr) + } + if loaded || !rep.Malformed || rep.Refused { + t.Errorf("%s in a *string blob record is damage, terminal for the batch (§3.1, §3.3): ok=%v report=%+v", dr.name, loaded, rep) + } + } + + // AND A *bytes BLOB HAS NO SUCH RULE, because it is bytes and never text + // (§3.1): the same bytes under the reserved `bytes` id load clean, which + // holds the content rule to the `string` id alone + _, ok, report, err = decodeOne(t, model, "Catalog", v, body(bytesType, []byte{'p', 'a', 0x00, 'c', 0xFF})) + if err != nil || !ok || !report.Silent() { + t.Errorf("the same bytes under the reserved bytes id load silently (§3.1): ok=%v err=%v report=%+v", ok, err, report) + } +} diff --git a/test/tables/message_blob_main.cpp b/test/tables/message_blob_main.cpp new file mode 100644 index 000000000..6b77f934a --- /dev/null +++ b/test/tables/message_blob_main.cpp @@ -0,0 +1,190 @@ +// THE MESSAGE FORM'S CONTENT RULE FOR A *string BLOB RECORD +// (docs/SPEC-TABLES.md §3, §3.1, §3.3). +// +// §3.1 states kinds `12` and `33`'s content rule MET AT A NODE: "A TEXT blob's +// CONTENT is refused on the same terms", so a `*string` blob whose bytes are +// not well-formed UTF-8, or which carries a zero byte, is damage and not data. +// §3.3 says a form-`2` body's content rules are §3's, unchanged in what they +// reject, and that what differs is only the recovery, which a bit stream does +// not have: the damage is TERMINAL for the batch. +// +// A `*bytes` blob has no such rule, because it is bytes and never text, and +// the row below that carries the same ill-formed bytes under the reserved +// `bytes` id holds the rule to the `string` id alone. +// +// The instrument is a batch forged by hand over `blobdemo`'s `Catalog`, whose +// numbering reaches a `*string` blob through `note` and a `*bytes` blob +// through `thumb`. A blob record is its own framing wherever it appears: the +// type reference, a length at thirty-two raw bits, the ALIGN, then the bytes +// verbatim. The program is green under the emitter this repository ships and +// red under one that drops the rule, which is the whole of +// `make tables-message-form-blob-negative-control`. + +#include +#include +#include +#include +#include + +#include "AssetsTable.h" + +static int failures = 0; + +static void row( bool ok, const char * name ) +{ + if ( ok ) + { + printf( "message blob: %s\n", name ); + return; + } + printf( "message blob FAILED: %s\n", name ); + failures++; +} + +// ONE BATCH OF ONE BODY carrying a NODE TABLE of one blob record and no field +// of its own (§3.1, §3.3): the form byte, the count as `M - 1`, the node +// table's own reference, the record count at thirty-two bits, the record's +// type reference, its length at thirty-two bits, the align, the bytes, the +// body's zero reference, and the pad to the byte boundary. A record is +// numbered and placed whether or not a slot names it, so the content rule is +// reached by the record alone. +static int64_t forge( uint8_t * out, int64_t capacity, const blobdemo::TableVocabulary & vocabulary, + uint64_t type_slot, const uint8_t * data, uint64_t length ) +{ + blobdemo::TableBitWriter w( out, capacity ); + w.put( 2, 8 ); + w.put( 0, 8 ); + w.put( blobdemo::kTableNodeTableFieldSlot, vocabulary.ref_bits ); + w.put( 1, 32 ); + w.put( type_slot, vocabulary.ref_bits ); + w.put( length, 32 ); + w.align(); + w.putbytes( data, (int64_t) length ); + w.put( 0, vocabulary.ref_bits ); + w.align(); + if ( w.overflow ) { return -1; } + return w.bits / 8; +} + +struct read_back +{ + bool loaded; + blobdemo::TableReport report; +}; + +static read_back load( const blobdemo::TableVocabulary & vocabulary, const uint8_t * batch, int64_t bytes ) +{ + read_back out; + out.loaded = false; + const int64_t need = blobdemo::CatalogLoadMeasure( vocabulary, batch, bytes, NULL ); + if ( need < 0 ) { return out; } + uint8_t * region = (uint8_t *) malloc( need > 0 ? (size_t) need : 1 ); + const blobdemo::Catalog * roots[1] = { NULL }; + int64_t count = 1; + blobdemo::CatalogLoadMessages( roots, &count, region, need, vocabulary, batch, bytes, &out.report ); + out.loaded = roots[0] != NULL && count == 1; + free( region ); + return out; +} + +int main() +{ + std::vector announcement( (size_t) blobdemo::AnnounceMeasure() ); + if ( blobdemo::Announce( announcement.data(), (int64_t) announcement.size() ) != (int64_t) announcement.size() ) + { + printf( "message blob: the announcement did not write\n" ); + return 2; + } + std::vector entries( (size_t) blobdemo::kTableMessageEntriesHere ); + blobdemo::TableVocabulary vocabulary( entries.data(), blobdemo::kTableMessageEntriesHere ); + if ( !blobdemo::AnnounceRead( vocabulary, announcement.data(), (int64_t) announcement.size(), NULL ) ) + { + printf( "message blob: this unit's own announcement was refused\n" ); + return 2; + } + + // THE TWO RESERVED BLOB IDS ride in the announcement's tail whether or not + // a root names them (§3.1), so the slots come off the vocabulary rather + // than out of a number this program spells + uint64_t string_slot = 0, bytes_slot = 0; + for ( int64_t at = 1; at <= vocabulary.count; at++ ) + { + const blobdemo::TableMessageEntry e = blobdemo::TableVocabularyEntryAt( vocabulary, at ); + if ( e.id == blobdemo::kTableStringTypeId ) { string_slot = (uint64_t) at; } + if ( e.id == blobdemo::kTableBytesTypeId ) { bytes_slot = (uint64_t) at; } + } + if ( string_slot == 0 || bytes_slot == 0 ) + { + printf( "message blob: this unit announces neither reserved blob id\n" ); + return 2; + } + + uint8_t batch[128]; + + // A WELL-FORMED BLOB IS ORDINARY, and the row is here first so a gate that + // went red by refusing everything is not one that holds the rule + { + const uint8_t data[8] = { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h' }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 8 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.loaded && !got.report.malformed && !got.report.refused && got.report.unknown == 0, + "a well-formed *string blob on a message body loads with a silent report" ); + } + + // A TRUNCATED SEQUENCE is a payload that is not text at any length (§3) + { + const uint8_t data[5] = { 'p', 'a', 'c', 'k', 0xC3 }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 5 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.report.malformed && !got.report.refused, + "a truncated UTF-8 sequence in a *string blob record is damage, and the batch ends there" ); + } + + // A ZERO BYTE AMONG THE BYTES is damage on the same rule (§3, §3.1) + { + const uint8_t data[5] = { 'p', 'a', 0x00, 'c', 'k' }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 5 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.report.malformed && !got.report.refused, + "a zero byte among a *string blob's bytes is damage, and the batch ends there" ); + } + + // AN OVERLONG ENCODING is ill-formed even though its bytes are a legal + // shape: 0xC0 0x80 spells U+0000 in two bytes + { + const uint8_t data[4] = { 'p', 0xC0, 0x80, 'k' }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 4 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.report.malformed && !got.report.refused, + "an overlong encoding in a *string blob record is damage, and the batch ends there" ); + } + + // A LONE LEAD BYTE UTF-8 NEVER SPELLS, which is the byte the pinned vector + // message_blob_ill_formed_text carries + { + const uint8_t data[8] = { 'a', 'b', 'c', 0xFF, 'e', 'f', 'g', 'h' }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 8 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.report.malformed && !got.report.refused, + "a lead byte UTF-8 never spells is damage, and the batch ends there" ); + } + + // AND A *bytes BLOB HAS NO SUCH RULE, because it is bytes and never text + // (§3.1): the same bytes under the reserved `bytes` id load clean, which + // holds the content rule to the `string` id alone + { + const uint8_t data[5] = { 'p', 'a', 0x00, 'c', 0xFF }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, bytes_slot, data, 5 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.loaded && !got.report.malformed && !got.report.refused, + "the same bytes under the reserved bytes id load with a silent report" ); + } + + if ( failures != 0 ) + { + printf( "message blob: %d row(s) red\n", failures ); + return 1; + } + printf( "message blob: a *string blob record on a message body carries kind 12's content rule\n" ); + return 0; +} From 97b9189799bf98584f08faa06c005ed145db638b Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 05:52:40 -0400 Subject: [PATCH 2/5] tables: a *string blob record on a message body carries the content rule (#632) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both engines read a form-2 blob record's bytes through the SAME function the FILE form reads one with, so there is one rule and no second copy. The C++ emitter: messagevariable.go's PASS TWO, where the record's bytes are already in hand. The align a blob record spends before its bytes leaves the span on a byte boundary, so the same pointer goes to the runtime's TableUtf8Valid and to the memcpy below it, and the line is emitted under rootReachesStringBlob, the same guard pointers.go emits the file form's under. The check does NOT ride in MessageRecordScan, which is the framing walk LoadMeasure shares: ill-formed content is sizeable, so a measure still answers the region the framing commands and only the decode refuses. The Go oracle: messagedecode.go's placement loop calls textValid on the record's bytes under the TString arm alone, which is decodenodes.go's line for the file form. What differs from the file form is only the recovery, which a bit stream does not have: where a file counts the record malformed and reads on with every slot naming it null, a batch ends there. One malformed counts, the bodies before it stand, and nothing after is read (§3.3). A *bytes blob keeps no rule, because it is bytes and never text (§3.1), and both gates carry the row that says so. Co-Authored-By: Claude Opus 5 --- internal/codegen/cpptable/messagevariable.go | 11 +++++++++++ internal/tablewire/messagedecode.go | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/internal/codegen/cpptable/messagevariable.go b/internal/codegen/cpptable/messagevariable.go index 17f1c08ff..dac8ae348 100644 --- a/internal/codegen/cpptable/messagevariable.go +++ b/internal/codegen/cpptable/messagevariable.go @@ -242,6 +242,17 @@ func (g *tableGen) emitVariableMessageSurface(st *ir.Struct) { g.pf(" if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId )\n {\n") g.pf(" uint64_t length = 0;\n") g.pf(" if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; }\n") + if g.rootReachesStringBlob(st) { + // A TEXT BLOB'S CONTENT IS REFUSED ON THE SAME TERMS as a kind 12 + // payload (docs/SPEC-TABLES.md §3.1), through the runtime's own + // TableUtf8Valid, which is what the FILE form reads one with in + // pointers.go. The align above already left the bytes on a byte + // boundary, so the span goes to the check as it goes to the memcpy + // below. What differs from the file form is only the RECOVERY, which + // a bit stream does not have: the damage is TERMINAL for the batch, + // one malformed counts, and the bodies before it stand (§3.3). + g.pf(" if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; }\n") + } g.pf(" if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); }\n") g.pf(" r.offset += (int64_t) length * 8;\n") g.pf(" continue;\n }\n") diff --git a/internal/tablewire/messagedecode.go b/internal/tablewire/messagedecode.go index cfb6a162f..d00371b30 100644 --- a/internal/tablewire/messagedecode.go +++ b/internal/tablewire/messagedecode.go @@ -360,6 +360,16 @@ func (d *bitDecoder) nodeTable(inst *tabletext.Instance, st *decodeState) bool { d.report.Unknown++ continue } + // A TEXT BLOB'S CONTENT IS REFUSED ON THE SAME TERMS as a kind 12 + // payload (§3.1), through the same textValid decodenodes.go reads + // the FILE form's record with, so there is one rule and no second + // copy. What differs is only the RECOVERY, which a bit stream does + // not have: the damage is TERMINAL for the batch, one malformed + // counts, and the bodies before it stand (§3.3). + if kind == ir.TString && !textValid(rec.blob) { + d.report.Malformed = true + return false + } st.nodes[i] = Node{Blob: &tabletext.Blob{Data: rec.blob}, Kind: kind} continue } From bdf8e56d463eab8a3ae07eab0006204365aef337 Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 05:52:52 -0400 Subject: [PATCH 3/5] tables: re-pin the zero-cost goldens the blob rule moves, and the six main left stale (#632) FOUR LINES ARE THIS BRANCH'S, one per emitted message reader whose root's numbering reaches a *string blob: blobs/AssetsTable.h twice, for Asset and for Catalog; maps/DocsTable.h once, for a map whose VALUE is a text buffer; and arms/GateTable.h once, for a blob reached only through a union arm. Each is the same line, the record's bytes handed to TableUtf8Valid. NO WIRE GOLDEN MOVED: the write side is untouched, and a read that refuses damage changes no byte a writer produces. THE REST WAS ALREADY STALE ON MAIN and `make tables-block-zero-cost` was red before this branch: e77093cd (#658) added the retain walk's `case 15: case 30:` and e123f1b2 (#662) moved the map entry readers' text path, and neither re-pinned tables/maps. maps/ChunksTable.h, RunsTable.h, SlotsTable.h and SpansTable.h carry nothing of this branch's at all, and CellsTable.h carries none of its four lines either. The gate now compares 117 Table sources byte-identical to their pins. Co-Authored-By: Claude Opus 5 --- testdata/golden/tables/arms/GateTable.h | 1 + testdata/golden/tables/blobs/AssetsTable.h | 2 ++ testdata/golden/tables/maps/CellsTable.h | 29 +++++++++++++++------ testdata/golden/tables/maps/ChunksTable.h | 15 +++++++++++ testdata/golden/tables/maps/DocsTable.h | 30 ++++++++++++++++------ testdata/golden/tables/maps/RunsTable.h | 15 +++++++++++ testdata/golden/tables/maps/SlotsTable.h | 15 +++++++++++ testdata/golden/tables/maps/SpansTable.h | 15 +++++++++++ 8 files changed, 106 insertions(+), 16 deletions(-) diff --git a/testdata/golden/tables/arms/GateTable.h b/testdata/golden/tables/arms/GateTable.h index 4913868fb..6e04cdf93 100644 --- a/testdata/golden/tables/arms/GateTable.h +++ b/testdata/golden/tables/arms/GateTable.h @@ -7185,6 +7185,7 @@ inline bool GateLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & { uint64_t length = 0; if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } r.offset += (int64_t) length * 8; continue; diff --git a/testdata/golden/tables/blobs/AssetsTable.h b/testdata/golden/tables/blobs/AssetsTable.h index 7b728be6b..47216b624 100644 --- a/testdata/golden/tables/blobs/AssetsTable.h +++ b/testdata/golden/tables/blobs/AssetsTable.h @@ -6720,6 +6720,7 @@ inline bool AssetLoadMessageBodyInto( TableBitReader & r, const TableVocabulary { uint64_t length = 0; if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } r.offset += (int64_t) length * 8; continue; @@ -7570,6 +7571,7 @@ inline bool CatalogLoadMessageBodyInto( TableBitReader & r, const TableVocabular { uint64_t length = 0; if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } r.offset += (int64_t) length * 8; continue; diff --git a/testdata/golden/tables/maps/CellsTable.h b/testdata/golden/tables/maps/CellsTable.h index 907c00422..92be183bf 100644 --- a/testdata/golden/tables/maps/CellsTable.h +++ b/testdata/golden/tables/maps/CellsTable.h @@ -3752,6 +3752,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4138,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -6480,15 +6495,13 @@ inline bool CellsRowsEntryLoadMessageBody( TableBitReader & r, const TableVocabu { uint64_t n = 0; if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } - int32_t kept = 0; - if ( n > (uint64_t) 8 ) { kept = 8; report->clamped++; } else { kept = (int32_t) n; } - for ( uint64_t i = 0; i < n; i++ ) - { - uint64_t by = 0; - if ( !r.get( by, 8 ) ) { report->malformed = true; return false; } - if ( (int32_t) i < kept ) { value.key[i] = (char) by; } - } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); value.key[kept] = 0; + r.offset += (int64_t) n * 8; value.key_length = kept; } break; diff --git a/testdata/golden/tables/maps/ChunksTable.h b/testdata/golden/tables/maps/ChunksTable.h index 6a1d6b153..7d7f7ce4f 100644 --- a/testdata/golden/tables/maps/ChunksTable.h +++ b/testdata/golden/tables/maps/ChunksTable.h @@ -3752,6 +3752,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4138,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; diff --git a/testdata/golden/tables/maps/DocsTable.h b/testdata/golden/tables/maps/DocsTable.h index acdfed2b6..98b4f6039 100644 --- a/testdata/golden/tables/maps/DocsTable.h +++ b/testdata/golden/tables/maps/DocsTable.h @@ -3752,6 +3752,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4138,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -6458,15 +6473,13 @@ inline bool DocsPagesEntryLoadMessageBody( TableBitReader & r, const TableVocabu { uint64_t n = 0; if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } - int32_t kept = 0; - if ( n > (uint64_t) 8 ) { kept = 8; report->clamped++; } else { kept = (int32_t) n; } - for ( uint64_t i = 0; i < n; i++ ) - { - uint64_t by = 0; - if ( !r.get( by, 8 ) ) { report->malformed = true; return false; } - if ( (int32_t) i < kept ) { value.key[i] = (char) by; } - } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); value.key[kept] = 0; + r.offset += (int64_t) n * 8; value.key_length = kept; } break; @@ -7941,6 +7954,7 @@ inline bool DocsLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & { uint64_t length = 0; if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } r.offset += (int64_t) length * 8; continue; diff --git a/testdata/golden/tables/maps/RunsTable.h b/testdata/golden/tables/maps/RunsTable.h index 2eb12b0c3..f841cd744 100644 --- a/testdata/golden/tables/maps/RunsTable.h +++ b/testdata/golden/tables/maps/RunsTable.h @@ -3753,6 +3753,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4139,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; diff --git a/testdata/golden/tables/maps/SlotsTable.h b/testdata/golden/tables/maps/SlotsTable.h index c2c0bed59..6d6198c3a 100644 --- a/testdata/golden/tables/maps/SlotsTable.h +++ b/testdata/golden/tables/maps/SlotsTable.h @@ -3753,6 +3753,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4139,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; diff --git a/testdata/golden/tables/maps/SpansTable.h b/testdata/golden/tables/maps/SpansTable.h index 580da8759..f7d9a3fe0 100644 --- a/testdata/golden/tables/maps/SpansTable.h +++ b/testdata/golden/tables/maps/SpansTable.h @@ -3753,6 +3753,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4139,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; From 5eff2d4f9139dc9cc39ed1a80df0bddc4c4991a5 Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 05:58:33 -0400 Subject: [PATCH 4/5] tables: two blades and two pinned vectors under the blob content rule (#632) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ONE BLADE AN ENGINE, each removing exactly one line and each naming what it turns red. message-blob-accepts-ill-formed drops textValid from the oracle's message placement loop; message-emitter-blob-accepts-ill-formed drops the emitted TableUtf8Valid call from messagevariable.go's PASS TWO. Each blade is drawn twice. Against the PAGE: the oracle's rides in MESSAGE_FORM_CONTROLS against TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule, and the emitter's in the new tables-message-form-blob-negative-control, which builds test/tables/message_blob_main.cpp against a regenerated tables/blobs. The true run of that program is the target's own first step, so the gate is green before the blade is drawn, and the sabotage reddens four rows and leaves the two silent-report rows standing. Against EACH OTHER: message_blob_ill_formed_text is blob_str8 re-encoded as a batch of one with the fourth byte of note's eight byte payload replaced by 0xFF, and message_blob_zero_byte is the same wire with 0x00 there instead, so the two separate the two halves of one check. Nothing else moves, so what the two readers answer differently is the content rule and nothing else. The *bytes record beside it carries bytes no UTF-8 rule would accept and is never checked, which is the vector's own control. tables-wire-fuzz-message-blob-oracle-negative-control and its leg twin replay both vectors alone and require red on each, with the reports mirrored: the leg says 0,0,0,0,0,true,read, the oracle says 0,0,0,0,0,false,read the leg says 0,0,0,0,0,false,read, the oracle says 0,0,0,0,0,true,read AND THE LEG GREW THE ROOT THE VECTORS NEED. blobdemo's Catalog is the only root on the fuzzer's roster whose numbering can PLACE a text blob record, so it is the only one whose wire can carry §3.1's rule at a node. Its AssetsTable.cpp was already in CONFORMANCE_SOURCES, so the leg costs one include and one MESSAGE_VARIABLE row, and the corpus pass now runs 45 roots rather than 44. Co-Authored-By: Claude Opus 5 --- Makefile | 111 +++++++++++++++++- test/tables/wire_fuzz_main.cpp | 7 ++ testdata/wire/tables/fuzz-vectors/INDEX.txt | 29 +++++ .../message_blob_ill_formed_text.bin | Bin 0 -> 38 bytes .../fuzz-vectors/message_blob_zero_byte.bin | Bin 0 -> 38 bytes tools/sabotage/message.go | 20 ++++ 6 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin create mode 100644 testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin diff --git a/Makefile b/Makefile index d62ce7ff6..6573fe829 100644 --- a/Makefile +++ b/Makefile @@ -4030,7 +4030,7 @@ tables-cpp-release: $(MAKE) tables-wire-fuzz-retain SEED=2 N=500000 .PHONY: tables-wire-fuzz-negative-control tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control -tables-wire-fuzz-negative-control: tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control tables-wire-fuzz-wide-text-negative-control tables-wire-fuzz-message-text-oracle-negative-control tables-wire-fuzz-message-text-leg-negative-control +tables-wire-fuzz-negative-control: tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control tables-wire-fuzz-wide-text-negative-control tables-wire-fuzz-message-text-oracle-negative-control tables-wire-fuzz-message-text-leg-negative-control tables-wire-fuzz-message-blob-oracle-negative-control tables-wire-fuzz-message-blob-leg-negative-control # THE CONTENT RULE ON KIND 33 (docs/SPEC-TABLES.md §3, §4): an unpaired # surrogate is DAMAGE, not data. The fuzzer's wide-text pass plants one at @@ -4301,10 +4301,11 @@ MESSAGE_FORM_CONTROLS := \ message-array-of-text-accepted:ir/tablemessage.go:TestAHostileShape \ message-skipped-variant-unresolved:internal/tablewire/messagedecode.go:TestAReferenceOfTheWrongSort \ message-text-accepts-ill-formed:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp \ - message-text-clamp-off-boundary:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp + message-text-clamp-off-boundary:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp \ + message-blob-accepts-ill-formed:internal/tablewire/messagedecode.go:TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule .PHONY: tables-message-form-negative-control -tables-message-form-negative-control: tables-message-form-emitter-negative-control tables-message-form-count-negative-control tables-message-form-text-negative-control +tables-message-form-negative-control: tables-message-form-emitter-negative-control tables-message-form-count-negative-control tables-message-form-text-negative-control tables-message-form-blob-negative-control @for row in $(MESSAGE_FORM_CONTROLS); do \ name=$${row%%:*}; rest=$${row#*:}; file=$${rest%%:*}; test=$${rest#*:}; \ $(MAKE) --no-print-directory tables-message-form-one-negative-control \ @@ -4420,6 +4421,42 @@ tables-message-form-text-negative-control: bin/schema test/tables/message_text_m $(call message_form_text_control,message-emitter-text-accepts-ill-formed,internal/codegen/cpptable/messageload.go) $(call message_form_text_control,message-emitter-text-clamp-off-boundary,internal/codegen/cpptable/messageload.go) +# AND THE SAME CONTENT RULE MET AT A NODE (docs/SPEC-TABLES.md §3.1, §3.3; +# schema#632): a `*string` blob record on a form-2 body is refused on the file +# form's own terms, and the damage is terminal for the batch. A writer produces +# no such record, so the instrument is a batch forged over `blobdemo`'s +# `Catalog`, whose numbering reaches a `*string` blob through `note` and a +# `*bytes` blob through `thumb`, and a program that reads the report back. The +# emitter's own copy of the rule is what the sabotage removes; the ORACLE's +# rides in MESSAGE_FORM_CONTROLS above. $(1) the sabotage, $(2) the emitter file. +define message_form_blob_control + @mkdir -p build/message-nc + @go run ./tools/sabotage -name $(1) -out build/message-nc/$(1).gotext $(2) + @printf '{"Replace":{"%s/$(2)":"%s/build/message-nc/$(1).gotext"}}\n' \ + "$(CURDIR)" "$(CURDIR)" > build/message-nc/$(1)-overlay.json + go build -overlay build/message-nc/$(1)-overlay.json -o build/message-nc/$(1)-schema ./cmd/schema + @rm -rf build/message-nc/$(1)-blobs && mkdir -p build/message-nc/$(1)-blobs + ./build/message-nc/$(1)-schema generate --lang cpp --out build/message-nc/$(1)-blobs tables/blobs + $(CXX) $(TABLES_CXXFLAGS) -Ibuild/message-nc/$(1)-blobs -Itest/tables -I$(SERIALIZE) \ + test/tables/message_blob_main.cpp build/message-nc/$(1)-blobs/AssetsTable.cpp \ + -o build/message-nc/$(1)-control + @if ./build/message-nc/$(1)-control > build/message-nc/$(1).log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the $(1) sabotage landed and the message blob rows stayed green"; \ + cat build/message-nc/$(1).log; exit 1; \ + fi + @cat build/message-nc/$(1).log + @echo "negative control ($(1)): the message form's blob rows go red" +endef + +.PHONY: tables-message-form-blob-negative-control +tables-message-form-blob-negative-control: bin/schema test/tables/message_blob_main.cpp build/tables-generated/.stamp + @mkdir -p build/message-nc + $(CXX) $(TABLES_CXXFLAGS) -Ibuild/tables-generated/blobs -Itest/tables -I$(SERIALIZE) \ + test/tables/message_blob_main.cpp build/tables-generated/blobs/AssetsTable.cpp \ + -o build/message-nc/blob-true + ./build/message-nc/blob-true + $(call message_form_blob_control,message-emitter-blob-accepts-ill-formed,internal/codegen/cpptable/messagevariable.go) + # THE NODE TYPE A ROOT CANNOT PLACE (docs/SPEC-TABLES.md §3.1, §6.5, §3.3), and # the vector message_node_type_unpointed is the red it closed. A node record is # a pointer's pointee, so a table no pointer below the root targets is a node @@ -4558,6 +4595,74 @@ tables-wire-fuzz-message-text-leg-negative-control: build/conformance-harness @grep -m1 "FAILED" $(MESSAGE_TEXT_LEG_NC)/log @echo "negative control: an emitted reader that accepts ill-formed message text turns the pinned vector RED" +# ILL-FORMED TEXT IN A *string BLOB RECORD ON A MESSAGE BODY +# (docs/SPEC-TABLES.md §3, §3.1, §3.3; schema#632), and the two vectors +# message_blob_ill_formed_text and message_blob_zero_byte are what hold the two +# engines to one answer on it. The blade in +# tables-message-form-blob-negative-control holds each engine to the PAGE; +# these two hold the engines to EACH OTHER, which is a different question and +# the one a differential fuzzer exists to ask: before the repair both readers +# placed the record, agreeing and agreeing wrongly, so nothing here could have +# gone red. Each control repairs one engine and takes the rule back out of the +# other, and the run must go red ON THE VECTOR. +# +# THE ASSERTION IS THE VECTOR REPLAYED ALONE, not a corpus pass, for the reason +# the node-type control names: an enumerated mutant reaches the same check and +# would name itself instead of the property. +MESSAGE_BLOB_VECTOR := testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin +MESSAGE_BLOB_ZERO_VECTOR := testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin + +MESSAGE_BLOB_ORACLE_NC := build/wire-fuzz-nc-message-blob-oracle +.PHONY: tables-wire-fuzz-message-blob-oracle-negative-control +tables-wire-fuzz-message-blob-oracle-negative-control: build/conformance-harness build/wire-fuzz-cpp + @rm -rf $(MESSAGE_BLOB_ORACLE_NC) && mkdir -p $(MESSAGE_BLOB_ORACLE_NC) + @go run ./tools/sabotage -name message-blob-accepts-ill-formed \ + -out $(MESSAGE_BLOB_ORACLE_NC)/messagedecode.go.txt internal/tablewire/messagedecode.go + @printf '{"Replace":{"%s/internal/tablewire/messagedecode.go":"%s/$(MESSAGE_BLOB_ORACLE_NC)/messagedecode.go.txt"}}\n' \ + "$(CURDIR)" "$(CURDIR)" > $(MESSAGE_BLOB_ORACLE_NC)/overlay.json + go build -overlay $(MESSAGE_BLOB_ORACLE_NC)/overlay.json -o $(MESSAGE_BLOB_ORACLE_NC)/harness ./test/conformance/harness + @for vector in $(MESSAGE_BLOB_VECTOR) $(MESSAGE_BLOB_ZERO_VECTOR); do \ + if $(MESSAGE_BLOB_ORACLE_NC)/harness wire-fuzz --driver ./build/wire-fuzz-cpp \ + --replay $$vector --unit blobdemo --root Catalog --message \ + --failed $(MESSAGE_BLOB_ORACLE_NC)/failed.bin > $(MESSAGE_BLOB_ORACLE_NC)/log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the oracle places an ill-formed *string blob again and $$vector stayed green"; \ + cat $(MESSAGE_BLOB_ORACLE_NC)/log; exit 1; \ + fi; \ + grep -q "$$(basename $$vector)" $(MESSAGE_BLOB_ORACLE_NC)/log || \ + { echo "NEGATIVE CONTROL FAILED: the wire fuzzer went red, but not on $$vector"; \ + cat $(MESSAGE_BLOB_ORACLE_NC)/log; exit 1; }; \ + grep -m1 "FAILED" $(MESSAGE_BLOB_ORACLE_NC)/log; \ + done + @echo "negative control: an oracle that places an ill-formed *string blob turns both pinned vectors RED" + +MESSAGE_BLOB_LEG_NC := build/wire-fuzz-nc-message-blob-leg +.PHONY: tables-wire-fuzz-message-blob-leg-negative-control +tables-wire-fuzz-message-blob-leg-negative-control: build/conformance-harness + @rm -rf $(MESSAGE_BLOB_LEG_NC) && mkdir -p $(MESSAGE_BLOB_LEG_NC) + @go run ./tools/sabotage -name message-emitter-blob-accepts-ill-formed \ + -out $(MESSAGE_BLOB_LEG_NC)/messagevariable.go.txt internal/codegen/cpptable/messagevariable.go + @printf '{"Replace":{"%s/internal/codegen/cpptable/messagevariable.go":"%s/$(MESSAGE_BLOB_LEG_NC)/messagevariable.go.txt"}}\n' \ + "$(CURDIR)" "$(CURDIR)" > $(MESSAGE_BLOB_LEG_NC)/overlay.json + go build -overlay $(MESSAGE_BLOB_LEG_NC)/overlay.json -o $(MESSAGE_BLOB_LEG_NC)/schema ./cmd/schema + $(call tables_generate,./$(MESSAGE_BLOB_LEG_NC)/schema,$(MESSAGE_BLOB_LEG_NC)/generated) + $(CXX) $(TABLES_CXXFLAGS) -O1 $(call tables_includes,$(MESSAGE_BLOB_LEG_NC)/generated) \ + test/tables/wire_fuzz_main.cpp \ + $(subst build/tables-generated/,$(MESSAGE_BLOB_LEG_NC)/generated/,$(CONFORMANCE_SOURCES)) \ + -o $(MESSAGE_BLOB_LEG_NC)/leg + @for vector in $(MESSAGE_BLOB_VECTOR) $(MESSAGE_BLOB_ZERO_VECTOR); do \ + if ./build/conformance-harness wire-fuzz --driver $(MESSAGE_BLOB_LEG_NC)/leg \ + --replay $$vector --unit blobdemo --root Catalog --message \ + --failed $(MESSAGE_BLOB_LEG_NC)/failed.bin > $(MESSAGE_BLOB_LEG_NC)/log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the emitted reader places an ill-formed *string blob again and $$vector stayed green"; \ + cat $(MESSAGE_BLOB_LEG_NC)/log; exit 1; \ + fi; \ + grep -q "$$(basename $$vector)" $(MESSAGE_BLOB_LEG_NC)/log || \ + { echo "NEGATIVE CONTROL FAILED: the wire fuzzer went red, but not on $$vector"; \ + cat $(MESSAGE_BLOB_LEG_NC)/log; exit 1; }; \ + grep -m1 "FAILED" $(MESSAGE_BLOB_LEG_NC)/log; \ + done + @echo "negative control: an emitted reader that places an ill-formed *string blob turns both pinned vectors RED" + # The GENERATED half of the data: the JSON text of every instance and the read # report of every evolution case, both from the compiler's own engine. .PHONY: conformance-generate diff --git a/test/tables/wire_fuzz_main.cpp b/test/tables/wire_fuzz_main.cpp index 50b30d71c..3d8db3ef3 100644 --- a/test/tables/wire_fuzz_main.cpp +++ b/test/tables/wire_fuzz_main.cpp @@ -49,6 +49,7 @@ #include "P2Table.h" #include "P3Table.h" #include "GraphTable.h" +#include "AssetsTable.h" // the BLOB unit (tables/blobs): a *string node record #include "MessagesTable.h" #include "StreamTable.h" #include "M1Table.h" @@ -404,6 +405,12 @@ static const Codec codecs[] = { MESSAGE( "vocab9demo", vocab9demo, Wide00 ), MESSAGE( "vocab9demo", vocab9demo, Wide19 ), MESSAGE_VARIABLE( "graphdemo", graphdemo, Scene ), + // AND THE BLOB UNIT UNDER THE MESSAGE FORM (docs/SPEC-TABLES.md §2.5, + // §3.1, §3.3): `Catalog`'s numbering reaches a *string blob through `note` + // and a *bytes blob through `thumb`, which is the only root on this roster + // that can PLACE a text blob record, and so the only one whose wire can + // carry the content rule §3.1 states at a node. + MESSAGE_VARIABLE( "blobdemo", blobdemo, Catalog ), // AND THE SAME VARIABLE-CLASS FILE ROOTS THROUGH THE RETAIN FAMILY // (docs/SPEC-TABLES.md §6.6). RETENTION IS THE VARIABLE CLASS'S: a // fixed-class root's LoadRetain is refused by name, and a form-2 diff --git a/testdata/wire/tables/fuzz-vectors/INDEX.txt b/testdata/wire/tables/fuzz-vectors/INDEX.txt index 0d13bfa7f..08451ff62 100644 --- a/testdata/wire/tables/fuzz-vectors/INDEX.txt +++ b/testdata/wire/tables/fuzz-vectors/INDEX.txt @@ -107,3 +107,32 @@ message_blob_node_unpointed graphdemo Scene testdata/wire/tables/fuzz-vectors/me # controls are the four in `make tables-message-form-negative-control`, two per # engine, each of which parts the leg from the oracle on this vector's name. message_ill_formed_text backenddemo StorePurchase testdata/wire/tables/fuzz-vectors/message_ill_formed_text.bin message + +# ILL-FORMED TEXT IN A *string BLOB RECORD ON A MESSAGE BODY (§3, §3.1, §3.3; +# schema#632). §3.1 states kinds `12` and `33`'s content rule MET AT A NODE, "A +# TEXT blob's CONTENT is refused on the same terms", and §3.3 says a form-`2` +# body's content rules are §3's, unchanged in what they reject: a `*string` +# blob whose bytes are not well-formed UTF-8 is DAMAGE, terminal for the batch +# here because a bit stream has no place to resume. NEITHER ENGINE CARRIED IT +# at the message site, where both carried it at the file site, so a record no +# reader should place loaded into both, agreeing, and agreeing wrongly, which +# is the one shape a differential fuzzer cannot see on its own. It became a red +# the moment one engine was repaired, and this vector is what holds the two +# together from here. +# +# The bytes are `blob_str8` re-encoded as a batch of one, with the fourth byte +# of `note`'s eight byte payload replaced by 0xFF, a lead byte UTF-8 never +# spells. Nothing else moves: the record's length, its align and the `*bytes` +# record beside it are the encoder's own, so what the two readers answer +# differently is the content rule and nothing else. The `*bytes` record is the +# vector's own control: it carries bytes no UTF-8 rule would accept and is +# never checked, because a `*bytes` blob is bytes and never text. +message_blob_ill_formed_text blobdemo Catalog testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin message + +# AND THE ZERO BYTE, which §3.1 names in the same sentence: a `*string` blob +# "whose bytes are not well-formed UTF-8 or which carries a zero byte". The +# bytes are the vector above's with 0x00 in place of the 0xFF, so the two +# separate the two halves of one check: a reader that rejected only the +# ill-formed sequence, or only the interior zero, goes red on one of them and +# not the other. +message_blob_zero_byte blobdemo Catalog testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin message diff --git a/testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin b/testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin new file mode 100644 index 0000000000000000000000000000000000000000..730d069ced5b413eb5df1004fecb5c1a62c3ae7a GIT binary patch literal 38 rcmZQ#&|+X3Q6L<89j3l=U~yd;pJB(*5lB7+YAvIGk7 literal 0 HcmV?d00001 diff --git a/testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin b/testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin new file mode 100644 index 0000000000000000000000000000000000000000..6f393facd026434f3e359a5d357096ececbc3938 GIT binary patch literal 38 rcmZQ#&|+X (uint64_t) %d ? (int32_t) %d : (int32_t) n%s; // SABOTAGED: the clamp is off the boundary\\n\", ind, sfx, sfx, f.Type.Size, f.Type.Size, sfx)\n", }}, + + // A TEXT BLOB'S CONTENT IS REFUSED ON THE SAME TERMS AS A KIND 12 PAYLOAD + // (docs/SPEC-TABLES.md §3.1), and a form-2 body's content rules are §3's + // (§3.3). One blade an engine: take the call out of the message site and + // the record a file reader refuses is placed by a message reader. + // + // THE ORACLE's, held by TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule + // and by the pinned vectors message_blob_ill_formed_text and + // message_blob_zero_byte. + "message-blob-accepts-ill-formed": {{ + old: "\t\t\tif kind == ir.TString && !textValid(rec.blob) {\n", + new: "\t\t\tif false && !textValid(rec.blob) { // SABOTAGED: the blob content rule is gone\n", + }}, + + // AND THE C++ EMITTER's, held by test/tables/message_blob_main.cpp and by + // the same two vectors. + "message-emitter-blob-accepts-ill-formed": {{ + old: "\t\tg.pf(\" if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; }\\n\")\n", + new: "\t\tg.pf(\" // SABOTAGED: the blob content rule is gone\\n\")\n", + }}, } From af26e8bb9eb607819492df5c76f9590ca443eb35 Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 05:59:03 -0400 Subject: [PATCH 5/5] =?UTF-8?q?docs:=20=C2=A73.3's=20test=20ledger=20names?= =?UTF-8?q?=20the=20content=20rule=20it=20now=20holds=20at=20a=20node=20(#?= =?UTF-8?q?632)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No sentence of §3 or §3.1 moves and none is added: §3.1 already says a text blob's CONTENT is refused on the same terms as a kind 12 payload, and §3.3 already says a form-2 body's content rules are §3's. What was stale is HELD BY TEST, which carries one bullet per rule with its red clause and carried none for this one. The bullet names the six rows the two engines' gates run, the *bytes row among them, and the three ways a leg goes red. Co-Authored-By: Claude Opus 5 --- docs/SPEC-TABLES.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/SPEC-TABLES.md b/docs/SPEC-TABLES.md index cb45b97c4..687cb28a9 100644 --- a/docs/SPEC-TABLES.md +++ b/docs/SPEC-TABLES.md @@ -5417,6 +5417,15 @@ entries announces about 5 KB once. lands whole and counts nothing. Red if a leg stores text the file form refuses, cuts a clamp inside a code point, keeps fewer bytes than the bound admits, or counts `clamped` on a payload that fits. +- **The same content rule met at a NODE.** A `*string` blob record on a + form-`2` body carrying a truncated sequence, one carrying a zero byte, one + carrying an overlong encoding and one carrying a lead byte UTF-8 never + spells, each damage and terminal for the batch on §3.1's own terms. Beside + them a well-formed blob, which loads with a silent report, and the same + ill-formed bytes under the reserved `bytes` id, which loads with a silent + report too, because a `*bytes` blob is bytes and never text. Red if a leg + places a record the file form refuses, refuses a record the file form places, + or reads a `*bytes` blob as text. - **The pad, and what follows it.** A batch whose trailing bits to the byte boundary are not zero, and a buffer carrying a whole batch and then a byte more. Red if a leg reads either clean.