diff --git a/Makefile b/Makefile index eb8cf8bb8..3ca076c9e 100644 --- a/Makefile +++ b/Makefile @@ -4030,7 +4030,7 @@ tables-cpp-release: $(MAKE) tables-wire-fuzz-retain SEED=2 N=500000 .PHONY: tables-wire-fuzz-negative-control tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control -tables-wire-fuzz-negative-control: tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control tables-wire-fuzz-wide-text-negative-control tables-wire-fuzz-message-text-oracle-negative-control tables-wire-fuzz-message-text-leg-negative-control +tables-wire-fuzz-negative-control: tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control tables-wire-fuzz-wide-text-negative-control tables-wire-fuzz-message-text-oracle-negative-control tables-wire-fuzz-message-text-leg-negative-control tables-wire-fuzz-message-blob-oracle-negative-control tables-wire-fuzz-message-blob-leg-negative-control # THE CONTENT RULE ON KIND 33 (docs/SPEC-TABLES.md §3, §4): an unpaired # surrogate is DAMAGE, not data. The fuzzer's wide-text pass plants one at @@ -4301,10 +4301,11 @@ MESSAGE_FORM_CONTROLS := \ message-array-of-text-accepted:ir/tablemessage.go:TestAHostileShape \ message-skipped-variant-unresolved:internal/tablewire/messagedecode.go:TestAReferenceOfTheWrongSort \ message-text-accepts-ill-formed:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp \ - message-text-clamp-off-boundary:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp + message-text-clamp-off-boundary:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp \ + message-blob-accepts-ill-formed:internal/tablewire/messagedecode.go:TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule .PHONY: tables-message-form-negative-control -tables-message-form-negative-control: tables-message-form-emitter-negative-control tables-message-form-count-negative-control tables-message-form-text-negative-control +tables-message-form-negative-control: tables-message-form-emitter-negative-control tables-message-form-count-negative-control tables-message-form-text-negative-control tables-message-form-blob-negative-control @for row in $(MESSAGE_FORM_CONTROLS); do \ name=$${row%%:*}; rest=$${row#*:}; file=$${rest%%:*}; test=$${rest#*:}; \ $(MAKE) --no-print-directory tables-message-form-one-negative-control \ @@ -4420,6 +4421,42 @@ tables-message-form-text-negative-control: bin/schema test/tables/message_text_m $(call message_form_text_control,message-emitter-text-accepts-ill-formed,internal/codegen/cpptable/messageload.go) $(call message_form_text_control,message-emitter-text-clamp-off-boundary,internal/codegen/cpptable/messageload.go) +# AND THE SAME CONTENT RULE MET AT A NODE (docs/SPEC-TABLES.md §3.1, §3.3; +# schema#632): a `*string` blob record on a form-2 body is refused on the file +# form's own terms, and the damage is terminal for the batch. A writer produces +# no such record, so the instrument is a batch forged over `blobdemo`'s +# `Catalog`, whose numbering reaches a `*string` blob through `note` and a +# `*bytes` blob through `thumb`, and a program that reads the report back. The +# emitter's own copy of the rule is what the sabotage removes; the ORACLE's +# rides in MESSAGE_FORM_CONTROLS above. $(1) the sabotage, $(2) the emitter file. +define message_form_blob_control + @mkdir -p build/message-nc + @go run ./tools/sabotage -name $(1) -out build/message-nc/$(1).gotext $(2) + @printf '{"Replace":{"%s/$(2)":"%s/build/message-nc/$(1).gotext"}}\n' \ + "$(CURDIR)" "$(CURDIR)" > build/message-nc/$(1)-overlay.json + go build -overlay build/message-nc/$(1)-overlay.json -o build/message-nc/$(1)-schema ./cmd/schema + @rm -rf build/message-nc/$(1)-blobs && mkdir -p build/message-nc/$(1)-blobs + ./build/message-nc/$(1)-schema generate --lang cpp --out build/message-nc/$(1)-blobs tables/blobs + $(CXX) $(TABLES_CXXFLAGS) -Ibuild/message-nc/$(1)-blobs -Itest/tables -I$(SERIALIZE) \ + test/tables/message_blob_main.cpp build/message-nc/$(1)-blobs/AssetsTable.cpp \ + -o build/message-nc/$(1)-control + @if ./build/message-nc/$(1)-control > build/message-nc/$(1).log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the $(1) sabotage landed and the message blob rows stayed green"; \ + cat build/message-nc/$(1).log; exit 1; \ + fi + @cat build/message-nc/$(1).log + @echo "negative control ($(1)): the message form's blob rows go red" +endef + +.PHONY: tables-message-form-blob-negative-control +tables-message-form-blob-negative-control: bin/schema test/tables/message_blob_main.cpp build/tables-generated/.stamp + @mkdir -p build/message-nc + $(CXX) $(TABLES_CXXFLAGS) -Ibuild/tables-generated/blobs -Itest/tables -I$(SERIALIZE) \ + test/tables/message_blob_main.cpp build/tables-generated/blobs/AssetsTable.cpp \ + -o build/message-nc/blob-true + ./build/message-nc/blob-true + $(call message_form_blob_control,message-emitter-blob-accepts-ill-formed,internal/codegen/cpptable/messagevariable.go) + # THE NODE TYPE A ROOT CANNOT PLACE (docs/SPEC-TABLES.md §3.1, §6.5, §3.3), and # the vector message_node_type_unpointed is the red it closed. A node record is # a pointer's pointee, so a table no pointer below the root targets is a node @@ -4558,6 +4595,74 @@ tables-wire-fuzz-message-text-leg-negative-control: build/conformance-harness @grep -m1 "FAILED" $(MESSAGE_TEXT_LEG_NC)/log @echo "negative control: an emitted reader that accepts ill-formed message text turns the pinned vector RED" +# ILL-FORMED TEXT IN A *string BLOB RECORD ON A MESSAGE BODY +# (docs/SPEC-TABLES.md §3, §3.1, §3.3; schema#632), and the two vectors +# message_blob_ill_formed_text and message_blob_zero_byte are what hold the two +# engines to one answer on it. The blade in +# tables-message-form-blob-negative-control holds each engine to the PAGE; +# these two hold the engines to EACH OTHER, which is a different question and +# the one a differential fuzzer exists to ask: before the repair both readers +# placed the record, agreeing and agreeing wrongly, so nothing here could have +# gone red. Each control repairs one engine and takes the rule back out of the +# other, and the run must go red ON THE VECTOR. +# +# THE ASSERTION IS THE VECTOR REPLAYED ALONE, not a corpus pass, for the reason +# the node-type control names: an enumerated mutant reaches the same check and +# would name itself instead of the property. +MESSAGE_BLOB_VECTOR := testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin +MESSAGE_BLOB_ZERO_VECTOR := testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin + +MESSAGE_BLOB_ORACLE_NC := build/wire-fuzz-nc-message-blob-oracle +.PHONY: tables-wire-fuzz-message-blob-oracle-negative-control +tables-wire-fuzz-message-blob-oracle-negative-control: build/conformance-harness build/wire-fuzz-cpp + @rm -rf $(MESSAGE_BLOB_ORACLE_NC) && mkdir -p $(MESSAGE_BLOB_ORACLE_NC) + @go run ./tools/sabotage -name message-blob-accepts-ill-formed \ + -out $(MESSAGE_BLOB_ORACLE_NC)/messagedecode.go.txt internal/tablewire/messagedecode.go + @printf '{"Replace":{"%s/internal/tablewire/messagedecode.go":"%s/$(MESSAGE_BLOB_ORACLE_NC)/messagedecode.go.txt"}}\n' \ + "$(CURDIR)" "$(CURDIR)" > $(MESSAGE_BLOB_ORACLE_NC)/overlay.json + go build -overlay $(MESSAGE_BLOB_ORACLE_NC)/overlay.json -o $(MESSAGE_BLOB_ORACLE_NC)/harness ./test/conformance/harness + @for vector in $(MESSAGE_BLOB_VECTOR) $(MESSAGE_BLOB_ZERO_VECTOR); do \ + if $(MESSAGE_BLOB_ORACLE_NC)/harness wire-fuzz --driver ./build/wire-fuzz-cpp \ + --replay $$vector --unit blobdemo --root Catalog --message \ + --failed $(MESSAGE_BLOB_ORACLE_NC)/failed.bin > $(MESSAGE_BLOB_ORACLE_NC)/log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the oracle places an ill-formed *string blob again and $$vector stayed green"; \ + cat $(MESSAGE_BLOB_ORACLE_NC)/log; exit 1; \ + fi; \ + grep -q "$$(basename $$vector)" $(MESSAGE_BLOB_ORACLE_NC)/log || \ + { echo "NEGATIVE CONTROL FAILED: the wire fuzzer went red, but not on $$vector"; \ + cat $(MESSAGE_BLOB_ORACLE_NC)/log; exit 1; }; \ + grep -m1 "FAILED" $(MESSAGE_BLOB_ORACLE_NC)/log; \ + done + @echo "negative control: an oracle that places an ill-formed *string blob turns both pinned vectors RED" + +MESSAGE_BLOB_LEG_NC := build/wire-fuzz-nc-message-blob-leg +.PHONY: tables-wire-fuzz-message-blob-leg-negative-control +tables-wire-fuzz-message-blob-leg-negative-control: build/conformance-harness + @rm -rf $(MESSAGE_BLOB_LEG_NC) && mkdir -p $(MESSAGE_BLOB_LEG_NC) + @go run ./tools/sabotage -name message-emitter-blob-accepts-ill-formed \ + -out $(MESSAGE_BLOB_LEG_NC)/messagevariable.go.txt internal/codegen/cpptable/messagevariable.go + @printf '{"Replace":{"%s/internal/codegen/cpptable/messagevariable.go":"%s/$(MESSAGE_BLOB_LEG_NC)/messagevariable.go.txt"}}\n' \ + "$(CURDIR)" "$(CURDIR)" > $(MESSAGE_BLOB_LEG_NC)/overlay.json + go build -overlay $(MESSAGE_BLOB_LEG_NC)/overlay.json -o $(MESSAGE_BLOB_LEG_NC)/schema ./cmd/schema + $(call tables_generate,./$(MESSAGE_BLOB_LEG_NC)/schema,$(MESSAGE_BLOB_LEG_NC)/generated) + $(CXX) $(TABLES_CXXFLAGS) -O1 $(call tables_includes,$(MESSAGE_BLOB_LEG_NC)/generated) \ + test/tables/wire_fuzz_main.cpp \ + $(subst build/tables-generated/,$(MESSAGE_BLOB_LEG_NC)/generated/,$(CONFORMANCE_SOURCES)) \ + -o $(MESSAGE_BLOB_LEG_NC)/leg + @for vector in $(MESSAGE_BLOB_VECTOR) $(MESSAGE_BLOB_ZERO_VECTOR); do \ + if ./build/conformance-harness wire-fuzz --driver $(MESSAGE_BLOB_LEG_NC)/leg \ + --replay $$vector --unit blobdemo --root Catalog --message \ + --failed $(MESSAGE_BLOB_LEG_NC)/failed.bin > $(MESSAGE_BLOB_LEG_NC)/log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the emitted reader places an ill-formed *string blob again and $$vector stayed green"; \ + cat $(MESSAGE_BLOB_LEG_NC)/log; exit 1; \ + fi; \ + grep -q "$$(basename $$vector)" $(MESSAGE_BLOB_LEG_NC)/log || \ + { echo "NEGATIVE CONTROL FAILED: the wire fuzzer went red, but not on $$vector"; \ + cat $(MESSAGE_BLOB_LEG_NC)/log; exit 1; }; \ + grep -m1 "FAILED" $(MESSAGE_BLOB_LEG_NC)/log; \ + done + @echo "negative control: an emitted reader that places an ill-formed *string blob turns both pinned vectors RED" + # The GENERATED half of the data: the JSON text of every instance and the read # report of every evolution case, both from the compiler's own engine. .PHONY: conformance-generate diff --git a/docs/SPEC-TABLES.md b/docs/SPEC-TABLES.md index e2a1b9578..e8f113f4a 100644 --- a/docs/SPEC-TABLES.md +++ b/docs/SPEC-TABLES.md @@ -5435,6 +5435,15 @@ entries announces about 5 KB once. lands whole and counts nothing. Red if a leg stores text the file form refuses, cuts a clamp inside a code point, keeps fewer bytes than the bound admits, or counts `clamped` on a payload that fits. +- **The same content rule met at a NODE.** A `*string` blob record on a + form-`2` body carrying a truncated sequence, one carrying a zero byte, one + carrying an overlong encoding and one carrying a lead byte UTF-8 never + spells, each damage and terminal for the batch on §3.1's own terms. Beside + them a well-formed blob, which loads with a silent report, and the same + ill-formed bytes under the reserved `bytes` id, which loads with a silent + report too, because a `*bytes` blob is bytes and never text. Red if a leg + places a record the file form refuses, refuses a record the file form places, + or reads a `*bytes` blob as text. - **The pad, and what follows it.** A batch whose trailing bits to the byte boundary are not zero, and a buffer carrying a whole batch and then a byte more. Red if a leg reads either clean. diff --git a/internal/codegen/cpptable/messagevariable.go b/internal/codegen/cpptable/messagevariable.go index 17f1c08ff..dac8ae348 100644 --- a/internal/codegen/cpptable/messagevariable.go +++ b/internal/codegen/cpptable/messagevariable.go @@ -242,6 +242,17 @@ func (g *tableGen) emitVariableMessageSurface(st *ir.Struct) { g.pf(" if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId )\n {\n") g.pf(" uint64_t length = 0;\n") g.pf(" if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; }\n") + if g.rootReachesStringBlob(st) { + // A TEXT BLOB'S CONTENT IS REFUSED ON THE SAME TERMS as a kind 12 + // payload (docs/SPEC-TABLES.md §3.1), through the runtime's own + // TableUtf8Valid, which is what the FILE form reads one with in + // pointers.go. The align above already left the bytes on a byte + // boundary, so the span goes to the check as it goes to the memcpy + // below. What differs from the file form is only the RECOVERY, which + // a bit stream does not have: the damage is TERMINAL for the batch, + // one malformed counts, and the bodies before it stand (§3.3). + g.pf(" if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; }\n") + } g.pf(" if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); }\n") g.pf(" r.offset += (int64_t) length * 8;\n") g.pf(" continue;\n }\n") diff --git a/internal/tablewire/messagedecode.go b/internal/tablewire/messagedecode.go index cfb6a162f..d00371b30 100644 --- a/internal/tablewire/messagedecode.go +++ b/internal/tablewire/messagedecode.go @@ -360,6 +360,16 @@ func (d *bitDecoder) nodeTable(inst *tabletext.Instance, st *decodeState) bool { d.report.Unknown++ continue } + // A TEXT BLOB'S CONTENT IS REFUSED ON THE SAME TERMS as a kind 12 + // payload (§3.1), through the same textValid decodenodes.go reads + // the FILE form's record with, so there is one rule and no second + // copy. What differs is only the RECOVERY, which a bit stream does + // not have: the damage is TERMINAL for the batch, one malformed + // counts, and the bodies before it stand (§3.3). + if kind == ir.TString && !textValid(rec.blob) { + d.report.Malformed = true + return false + } st.nodes[i] = Node{Blob: &tabletext.Blob{Data: rec.blob}, Kind: kind} continue } diff --git a/test/conformance/harness/messagerules_test.go b/test/conformance/harness/messagerules_test.go index b1dd2c4f5..2eddd5780 100644 --- a/test/conformance/harness/messagerules_test.go +++ b/test/conformance/harness/messagerules_test.go @@ -1512,3 +1512,81 @@ func TestTheMessageFormsTextContentRuleAndClamp(t *testing.T) { t.Errorf("a payload at the bound lands whole: kept %d bytes, %q", len(got), got) } } + +// TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule: kinds `12` and +// `33`'s content rule MET AT A NODE (docs/SPEC-TABLES.md §3.1) over a form-`2` +// body (§3.3). §3.1 refuses a text blob's CONTENT on the file form's own +// terms, and §3.3 says a form-`2` body's content rules are §3's, unchanged in +// what they reject: a `*string` blob whose bytes are not well-formed UTF-8, or +// which carries a zero byte, is DAMAGE and not data. What differs is only the +// recovery, which a bit stream does not have, so the damage is TERMINAL for +// the batch. Red if the message path places a blob the file path refuses. +// +// The instrument is `blobdemo`'s `Catalog`, whose numbering reaches a +// `*string` blob through `note` and a `*bytes` blob through `thumb`. A blob +// record is its own framing wherever it appears: the type reference, a length +// at thirty-two raw bits, the ALIGN, then the bytes verbatim. A record is +// numbered and placed whether or not a slot names it, so the rule is reached +// by the record alone. +func TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule(t *testing.T) { + _, _, u := corpus(t) + unit, err := u.get("blobdemo") + if err != nil { + t.Fatal(err) + } + model := tabletext.NewModel(unit) + v := vocabularyOf(t, unit) + // THE RESERVED IDS RIDE IN THE ANNOUNCEMENT'S TAIL whether or not a root + // names them (§3.1, §3.3), each a kind-0 entry that frames nothing + node := slotOf(t, v, ir.TableNodeWireId, 0) + stringType := slotOf(t, v, ir.StringWireTypeId, 0) + bytesType := slotOf(t, v, ir.BytesWireTypeId, 0) + body := func(typeSlot uint64, data []byte) []byte { + w := &bitw{} + w.put(node, v.RefBits()) + w.put(1, 32) + w.put(typeSlot, v.RefBits()) + w.put(uint64(len(data)), 32) + w.align() + w.bytes(data) + w.put(0, v.RefBits()) + return batchOf(w) + } + + // A WELL-FORMED BLOB IS ORDINARY, and the row is here first so a gate that + // went red by refusing everything is not one that holds the rule + _, ok, report, err := decodeOne(t, model, "Catalog", v, body(stringType, []byte("abcdefgh"))) + if err != nil || !ok || !report.Silent() { + t.Fatalf("a well-formed *string blob on a message body loads silently: ok=%v err=%v report=%+v", ok, err, report) + } + + // ILL-FORMED CONTENT IS DAMAGE AND IT IS TERMINAL: a truncated sequence, a + // zero byte, an overlong encoding and a lead byte UTF-8 never spells are + // each a payload that is not text at whatever length it arrived at + damaged := []struct { + name string + data []byte + }{ + {"a truncated sequence", []byte{'p', 'a', 'c', 'k', 0xC3}}, + {"a zero byte among the bytes", []byte{'p', 'a', 0x00, 'c', 'k'}}, + {"an overlong encoding", []byte{'p', 0xC0, 0x80, 'k'}}, + {"a lead byte UTF-8 never spells", []byte{'a', 'b', 'c', 0xFF, 'e', 'f', 'g', 'h'}}, + } + for _, dr := range damaged { + _, loaded, rep, derr := decodeOne(t, model, "Catalog", v, body(stringType, dr.data)) + if derr != nil { + t.Fatalf("%s: the decode errored: %v", dr.name, derr) + } + if loaded || !rep.Malformed || rep.Refused { + t.Errorf("%s in a *string blob record is damage, terminal for the batch (§3.1, §3.3): ok=%v report=%+v", dr.name, loaded, rep) + } + } + + // AND A *bytes BLOB HAS NO SUCH RULE, because it is bytes and never text + // (§3.1): the same bytes under the reserved `bytes` id load clean, which + // holds the content rule to the `string` id alone + _, ok, report, err = decodeOne(t, model, "Catalog", v, body(bytesType, []byte{'p', 'a', 0x00, 'c', 0xFF})) + if err != nil || !ok || !report.Silent() { + t.Errorf("the same bytes under the reserved bytes id load silently (§3.1): ok=%v err=%v report=%+v", ok, err, report) + } +} diff --git a/test/tables/message_blob_main.cpp b/test/tables/message_blob_main.cpp new file mode 100644 index 000000000..6b77f934a --- /dev/null +++ b/test/tables/message_blob_main.cpp @@ -0,0 +1,190 @@ +// THE MESSAGE FORM'S CONTENT RULE FOR A *string BLOB RECORD +// (docs/SPEC-TABLES.md §3, §3.1, §3.3). +// +// §3.1 states kinds `12` and `33`'s content rule MET AT A NODE: "A TEXT blob's +// CONTENT is refused on the same terms", so a `*string` blob whose bytes are +// not well-formed UTF-8, or which carries a zero byte, is damage and not data. +// §3.3 says a form-`2` body's content rules are §3's, unchanged in what they +// reject, and that what differs is only the recovery, which a bit stream does +// not have: the damage is TERMINAL for the batch. +// +// A `*bytes` blob has no such rule, because it is bytes and never text, and +// the row below that carries the same ill-formed bytes under the reserved +// `bytes` id holds the rule to the `string` id alone. +// +// The instrument is a batch forged by hand over `blobdemo`'s `Catalog`, whose +// numbering reaches a `*string` blob through `note` and a `*bytes` blob +// through `thumb`. A blob record is its own framing wherever it appears: the +// type reference, a length at thirty-two raw bits, the ALIGN, then the bytes +// verbatim. The program is green under the emitter this repository ships and +// red under one that drops the rule, which is the whole of +// `make tables-message-form-blob-negative-control`. + +#include +#include +#include +#include +#include + +#include "AssetsTable.h" + +static int failures = 0; + +static void row( bool ok, const char * name ) +{ + if ( ok ) + { + printf( "message blob: %s\n", name ); + return; + } + printf( "message blob FAILED: %s\n", name ); + failures++; +} + +// ONE BATCH OF ONE BODY carrying a NODE TABLE of one blob record and no field +// of its own (§3.1, §3.3): the form byte, the count as `M - 1`, the node +// table's own reference, the record count at thirty-two bits, the record's +// type reference, its length at thirty-two bits, the align, the bytes, the +// body's zero reference, and the pad to the byte boundary. A record is +// numbered and placed whether or not a slot names it, so the content rule is +// reached by the record alone. +static int64_t forge( uint8_t * out, int64_t capacity, const blobdemo::TableVocabulary & vocabulary, + uint64_t type_slot, const uint8_t * data, uint64_t length ) +{ + blobdemo::TableBitWriter w( out, capacity ); + w.put( 2, 8 ); + w.put( 0, 8 ); + w.put( blobdemo::kTableNodeTableFieldSlot, vocabulary.ref_bits ); + w.put( 1, 32 ); + w.put( type_slot, vocabulary.ref_bits ); + w.put( length, 32 ); + w.align(); + w.putbytes( data, (int64_t) length ); + w.put( 0, vocabulary.ref_bits ); + w.align(); + if ( w.overflow ) { return -1; } + return w.bits / 8; +} + +struct read_back +{ + bool loaded; + blobdemo::TableReport report; +}; + +static read_back load( const blobdemo::TableVocabulary & vocabulary, const uint8_t * batch, int64_t bytes ) +{ + read_back out; + out.loaded = false; + const int64_t need = blobdemo::CatalogLoadMeasure( vocabulary, batch, bytes, NULL ); + if ( need < 0 ) { return out; } + uint8_t * region = (uint8_t *) malloc( need > 0 ? (size_t) need : 1 ); + const blobdemo::Catalog * roots[1] = { NULL }; + int64_t count = 1; + blobdemo::CatalogLoadMessages( roots, &count, region, need, vocabulary, batch, bytes, &out.report ); + out.loaded = roots[0] != NULL && count == 1; + free( region ); + return out; +} + +int main() +{ + std::vector announcement( (size_t) blobdemo::AnnounceMeasure() ); + if ( blobdemo::Announce( announcement.data(), (int64_t) announcement.size() ) != (int64_t) announcement.size() ) + { + printf( "message blob: the announcement did not write\n" ); + return 2; + } + std::vector entries( (size_t) blobdemo::kTableMessageEntriesHere ); + blobdemo::TableVocabulary vocabulary( entries.data(), blobdemo::kTableMessageEntriesHere ); + if ( !blobdemo::AnnounceRead( vocabulary, announcement.data(), (int64_t) announcement.size(), NULL ) ) + { + printf( "message blob: this unit's own announcement was refused\n" ); + return 2; + } + + // THE TWO RESERVED BLOB IDS ride in the announcement's tail whether or not + // a root names them (§3.1), so the slots come off the vocabulary rather + // than out of a number this program spells + uint64_t string_slot = 0, bytes_slot = 0; + for ( int64_t at = 1; at <= vocabulary.count; at++ ) + { + const blobdemo::TableMessageEntry e = blobdemo::TableVocabularyEntryAt( vocabulary, at ); + if ( e.id == blobdemo::kTableStringTypeId ) { string_slot = (uint64_t) at; } + if ( e.id == blobdemo::kTableBytesTypeId ) { bytes_slot = (uint64_t) at; } + } + if ( string_slot == 0 || bytes_slot == 0 ) + { + printf( "message blob: this unit announces neither reserved blob id\n" ); + return 2; + } + + uint8_t batch[128]; + + // A WELL-FORMED BLOB IS ORDINARY, and the row is here first so a gate that + // went red by refusing everything is not one that holds the rule + { + const uint8_t data[8] = { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h' }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 8 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.loaded && !got.report.malformed && !got.report.refused && got.report.unknown == 0, + "a well-formed *string blob on a message body loads with a silent report" ); + } + + // A TRUNCATED SEQUENCE is a payload that is not text at any length (§3) + { + const uint8_t data[5] = { 'p', 'a', 'c', 'k', 0xC3 }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 5 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.report.malformed && !got.report.refused, + "a truncated UTF-8 sequence in a *string blob record is damage, and the batch ends there" ); + } + + // A ZERO BYTE AMONG THE BYTES is damage on the same rule (§3, §3.1) + { + const uint8_t data[5] = { 'p', 'a', 0x00, 'c', 'k' }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 5 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.report.malformed && !got.report.refused, + "a zero byte among a *string blob's bytes is damage, and the batch ends there" ); + } + + // AN OVERLONG ENCODING is ill-formed even though its bytes are a legal + // shape: 0xC0 0x80 spells U+0000 in two bytes + { + const uint8_t data[4] = { 'p', 0xC0, 0x80, 'k' }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 4 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.report.malformed && !got.report.refused, + "an overlong encoding in a *string blob record is damage, and the batch ends there" ); + } + + // A LONE LEAD BYTE UTF-8 NEVER SPELLS, which is the byte the pinned vector + // message_blob_ill_formed_text carries + { + const uint8_t data[8] = { 'a', 'b', 'c', 0xFF, 'e', 'f', 'g', 'h' }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, string_slot, data, 8 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.report.malformed && !got.report.refused, + "a lead byte UTF-8 never spells is damage, and the batch ends there" ); + } + + // AND A *bytes BLOB HAS NO SUCH RULE, because it is bytes and never text + // (§3.1): the same bytes under the reserved `bytes` id load clean, which + // holds the content rule to the `string` id alone + { + const uint8_t data[5] = { 'p', 'a', 0x00, 'c', 0xFF }; + const int64_t bytes = forge( batch, sizeof( batch ), vocabulary, bytes_slot, data, 5 ); + const read_back got = load( vocabulary, batch, bytes ); + row( got.loaded && !got.report.malformed && !got.report.refused, + "the same bytes under the reserved bytes id load with a silent report" ); + } + + if ( failures != 0 ) + { + printf( "message blob: %d row(s) red\n", failures ); + return 1; + } + printf( "message blob: a *string blob record on a message body carries kind 12's content rule\n" ); + return 0; +} diff --git a/test/tables/wire_fuzz_main.cpp b/test/tables/wire_fuzz_main.cpp index 50b30d71c..3d8db3ef3 100644 --- a/test/tables/wire_fuzz_main.cpp +++ b/test/tables/wire_fuzz_main.cpp @@ -49,6 +49,7 @@ #include "P2Table.h" #include "P3Table.h" #include "GraphTable.h" +#include "AssetsTable.h" // the BLOB unit (tables/blobs): a *string node record #include "MessagesTable.h" #include "StreamTable.h" #include "M1Table.h" @@ -404,6 +405,12 @@ static const Codec codecs[] = { MESSAGE( "vocab9demo", vocab9demo, Wide00 ), MESSAGE( "vocab9demo", vocab9demo, Wide19 ), MESSAGE_VARIABLE( "graphdemo", graphdemo, Scene ), + // AND THE BLOB UNIT UNDER THE MESSAGE FORM (docs/SPEC-TABLES.md §2.5, + // §3.1, §3.3): `Catalog`'s numbering reaches a *string blob through `note` + // and a *bytes blob through `thumb`, which is the only root on this roster + // that can PLACE a text blob record, and so the only one whose wire can + // carry the content rule §3.1 states at a node. + MESSAGE_VARIABLE( "blobdemo", blobdemo, Catalog ), // AND THE SAME VARIABLE-CLASS FILE ROOTS THROUGH THE RETAIN FAMILY // (docs/SPEC-TABLES.md §6.6). RETENTION IS THE VARIABLE CLASS'S: a // fixed-class root's LoadRetain is refused by name, and a form-2 diff --git a/testdata/golden/tables/arms/GateTable.h b/testdata/golden/tables/arms/GateTable.h index 4913868fb..6e04cdf93 100644 --- a/testdata/golden/tables/arms/GateTable.h +++ b/testdata/golden/tables/arms/GateTable.h @@ -7185,6 +7185,7 @@ inline bool GateLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & { uint64_t length = 0; if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } r.offset += (int64_t) length * 8; continue; diff --git a/testdata/golden/tables/blobs/AssetsTable.h b/testdata/golden/tables/blobs/AssetsTable.h index 7b728be6b..47216b624 100644 --- a/testdata/golden/tables/blobs/AssetsTable.h +++ b/testdata/golden/tables/blobs/AssetsTable.h @@ -6720,6 +6720,7 @@ inline bool AssetLoadMessageBodyInto( TableBitReader & r, const TableVocabulary { uint64_t length = 0; if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } r.offset += (int64_t) length * 8; continue; @@ -7570,6 +7571,7 @@ inline bool CatalogLoadMessageBodyInto( TableBitReader & r, const TableVocabular { uint64_t length = 0; if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } r.offset += (int64_t) length * 8; continue; diff --git a/testdata/golden/tables/maps/DocsTable.h b/testdata/golden/tables/maps/DocsTable.h index 00d6087fb..da13c5c5d 100644 --- a/testdata/golden/tables/maps/DocsTable.h +++ b/testdata/golden/tables/maps/DocsTable.h @@ -7966,6 +7966,7 @@ inline bool DocsLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & { uint64_t length = 0; if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } r.offset += (int64_t) length * 8; continue; diff --git a/testdata/wire/tables/fuzz-vectors/INDEX.txt b/testdata/wire/tables/fuzz-vectors/INDEX.txt index 0d13bfa7f..08451ff62 100644 --- a/testdata/wire/tables/fuzz-vectors/INDEX.txt +++ b/testdata/wire/tables/fuzz-vectors/INDEX.txt @@ -107,3 +107,32 @@ message_blob_node_unpointed graphdemo Scene testdata/wire/tables/fuzz-vectors/me # controls are the four in `make tables-message-form-negative-control`, two per # engine, each of which parts the leg from the oracle on this vector's name. message_ill_formed_text backenddemo StorePurchase testdata/wire/tables/fuzz-vectors/message_ill_formed_text.bin message + +# ILL-FORMED TEXT IN A *string BLOB RECORD ON A MESSAGE BODY (§3, §3.1, §3.3; +# schema#632). §3.1 states kinds `12` and `33`'s content rule MET AT A NODE, "A +# TEXT blob's CONTENT is refused on the same terms", and §3.3 says a form-`2` +# body's content rules are §3's, unchanged in what they reject: a `*string` +# blob whose bytes are not well-formed UTF-8 is DAMAGE, terminal for the batch +# here because a bit stream has no place to resume. NEITHER ENGINE CARRIED IT +# at the message site, where both carried it at the file site, so a record no +# reader should place loaded into both, agreeing, and agreeing wrongly, which +# is the one shape a differential fuzzer cannot see on its own. It became a red +# the moment one engine was repaired, and this vector is what holds the two +# together from here. +# +# The bytes are `blob_str8` re-encoded as a batch of one, with the fourth byte +# of `note`'s eight byte payload replaced by 0xFF, a lead byte UTF-8 never +# spells. Nothing else moves: the record's length, its align and the `*bytes` +# record beside it are the encoder's own, so what the two readers answer +# differently is the content rule and nothing else. The `*bytes` record is the +# vector's own control: it carries bytes no UTF-8 rule would accept and is +# never checked, because a `*bytes` blob is bytes and never text. +message_blob_ill_formed_text blobdemo Catalog testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin message + +# AND THE ZERO BYTE, which §3.1 names in the same sentence: a `*string` blob +# "whose bytes are not well-formed UTF-8 or which carries a zero byte". The +# bytes are the vector above's with 0x00 in place of the 0xFF, so the two +# separate the two halves of one check: a reader that rejected only the +# ill-formed sequence, or only the interior zero, goes red on one of them and +# not the other. +message_blob_zero_byte blobdemo Catalog testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin message diff --git a/testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin b/testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin new file mode 100644 index 000000000..730d069ce Binary files /dev/null and b/testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin differ diff --git a/testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin b/testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin new file mode 100644 index 000000000..6f393facd Binary files /dev/null and b/testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin differ diff --git a/tools/sabotage/message.go b/tools/sabotage/message.go index bd6933db9..f27c26950 100644 --- a/tools/sabotage/message.go +++ b/tools/sabotage/message.go @@ -405,4 +405,24 @@ var messageTextSabotages = map[string][]edit{ old: "\t\tg.pf(\"%s const int32_t kept%s = (int32_t) TableUtf8Clamp( text%s, n%s, %d );\\n\", ind, sfx, sfx, sfx, f.Type.Size)\n", new: "\t\tg.pf(\"%s const int32_t kept%s = n%s > (uint64_t) %d ? (int32_t) %d : (int32_t) n%s; // SABOTAGED: the clamp is off the boundary\\n\", ind, sfx, sfx, f.Type.Size, f.Type.Size, sfx)\n", }}, + + // A TEXT BLOB'S CONTENT IS REFUSED ON THE SAME TERMS AS A KIND 12 PAYLOAD + // (docs/SPEC-TABLES.md §3.1), and a form-2 body's content rules are §3's + // (§3.3). One blade an engine: take the call out of the message site and + // the record a file reader refuses is placed by a message reader. + // + // THE ORACLE's, held by TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule + // and by the pinned vectors message_blob_ill_formed_text and + // message_blob_zero_byte. + "message-blob-accepts-ill-formed": {{ + old: "\t\t\tif kind == ir.TString && !textValid(rec.blob) {\n", + new: "\t\t\tif false && !textValid(rec.blob) { // SABOTAGED: the blob content rule is gone\n", + }}, + + // AND THE C++ EMITTER's, held by test/tables/message_blob_main.cpp and by + // the same two vectors. + "message-emitter-blob-accepts-ill-formed": {{ + old: "\t\tg.pf(\" if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; }\\n\")\n", + new: "\t\tg.pf(\" // SABOTAGED: the blob content rule is gone\\n\")\n", + }}, }