tables: a blob string on a message body carries the content rule (#632)

Makefile

@@ -4030,7 +4030,7 @@ tables-cpp-release:
 	$(MAKE) tables-wire-fuzz-retain SEED=2 N=500000
 
 .PHONY: tables-wire-fuzz-negative-control tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control
-tables-wire-fuzz-negative-control: tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control tables-wire-fuzz-wide-text-negative-control tables-wire-fuzz-message-text-oracle-negative-control tables-wire-fuzz-message-text-leg-negative-control
+tables-wire-fuzz-negative-control: tables-wire-fuzz-length-negative-control tables-wire-fuzz-index-negative-control tables-wire-fuzz-arm-width-negative-control tables-wire-fuzz-arm-terminator-negative-control tables-wire-fuzz-oracle-negative-control tables-wire-fuzz-node-type-negative-control tables-wire-fuzz-blob-node-negative-control tables-wire-fuzz-wide-text-negative-control tables-wire-fuzz-message-text-oracle-negative-control tables-wire-fuzz-message-text-leg-negative-control tables-wire-fuzz-message-blob-oracle-negative-control tables-wire-fuzz-message-blob-leg-negative-control
 
 # THE CONTENT RULE ON KIND 33 (docs/SPEC-TABLES.md §3, §4): an unpaired
 # surrogate is DAMAGE, not data. The fuzzer's wide-text pass plants one at
@@ -4301,10 +4301,11 @@ MESSAGE_FORM_CONTROLS := \
 	message-array-of-text-accepted:ir/tablemessage.go:TestAHostileShape \
 	message-skipped-variant-unresolved:internal/tablewire/messagedecode.go:TestAReferenceOfTheWrongSort \
 	message-text-accepts-ill-formed:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp \
-	message-text-clamp-off-boundary:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp
+	message-text-clamp-off-boundary:internal/tablewire/messagedecode.go:TestTheMessageFormsTextContentRuleAndClamp \
+	message-blob-accepts-ill-formed:internal/tablewire/messagedecode.go:TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule
 
 .PHONY: tables-message-form-negative-control
-tables-message-form-negative-control: tables-message-form-emitter-negative-control tables-message-form-count-negative-control tables-message-form-text-negative-control
+tables-message-form-negative-control: tables-message-form-emitter-negative-control tables-message-form-count-negative-control tables-message-form-text-negative-control tables-message-form-blob-negative-control
 	@for row in $(MESSAGE_FORM_CONTROLS); do \
 		name=$${row%%:*}; rest=$${row#*:}; file=$${rest%%:*}; test=$${rest#*:}; \
 		$(MAKE) --no-print-directory tables-message-form-one-negative-control \
@@ -4420,6 +4421,42 @@ tables-message-form-text-negative-control: bin/schema test/tables/message_text_m
 	$(call message_form_text_control,message-emitter-text-accepts-ill-formed,internal/codegen/cpptable/messageload.go)
 	$(call message_form_text_control,message-emitter-text-clamp-off-boundary,internal/codegen/cpptable/messageload.go)
 
+# AND THE SAME CONTENT RULE MET AT A NODE (docs/SPEC-TABLES.md §3.1, §3.3;
+# schema#632): a `*string` blob record on a form-2 body is refused on the file
+# form's own terms, and the damage is terminal for the batch. A writer produces
+# no such record, so the instrument is a batch forged over `blobdemo`'s
+# `Catalog`, whose numbering reaches a `*string` blob through `note` and a
+# `*bytes` blob through `thumb`, and a program that reads the report back. The
+# emitter's own copy of the rule is what the sabotage removes; the ORACLE's
+# rides in MESSAGE_FORM_CONTROLS above. $(1) the sabotage, $(2) the emitter file.
+define message_form_blob_control
+	@mkdir -p build/message-nc
+	@go run ./tools/sabotage -name $(1) -out build/message-nc/$(1).gotext $(2)
+	@printf '{"Replace":{"%s/$(2)":"%s/build/message-nc/$(1).gotext"}}\n' \
+		"$(CURDIR)" "$(CURDIR)" > build/message-nc/$(1)-overlay.json
+	go build -overlay build/message-nc/$(1)-overlay.json -o build/message-nc/$(1)-schema ./cmd/schema
+	@rm -rf build/message-nc/$(1)-blobs && mkdir -p build/message-nc/$(1)-blobs
+	./build/message-nc/$(1)-schema generate --lang cpp --out build/message-nc/$(1)-blobs tables/blobs
+	$(CXX) $(TABLES_CXXFLAGS) -Ibuild/message-nc/$(1)-blobs -Itest/tables -I$(SERIALIZE) \
+		test/tables/message_blob_main.cpp build/message-nc/$(1)-blobs/AssetsTable.cpp \
+		-o build/message-nc/$(1)-control
+	@if ./build/message-nc/$(1)-control > build/message-nc/$(1).log 2>&1; then \
+		echo "NEGATIVE CONTROL FAILED: the $(1) sabotage landed and the message blob rows stayed green"; \
+		cat build/message-nc/$(1).log; exit 1; \
+	fi
+	@cat build/message-nc/$(1).log
+	@echo "negative control ($(1)): the message form's blob rows go red"
+endef
+
+.PHONY: tables-message-form-blob-negative-control
+tables-message-form-blob-negative-control: bin/schema test/tables/message_blob_main.cpp build/tables-generated/.stamp
+	@mkdir -p build/message-nc
+	$(CXX) $(TABLES_CXXFLAGS) -Ibuild/tables-generated/blobs -Itest/tables -I$(SERIALIZE) \
+		test/tables/message_blob_main.cpp build/tables-generated/blobs/AssetsTable.cpp \
+		-o build/message-nc/blob-true
+	./build/message-nc/blob-true
+	$(call message_form_blob_control,message-emitter-blob-accepts-ill-formed,internal/codegen/cpptable/messagevariable.go)
+
 # THE NODE TYPE A ROOT CANNOT PLACE (docs/SPEC-TABLES.md §3.1, §6.5, §3.3), and
 # the vector message_node_type_unpointed is the red it closed. A node record is
 # a pointer's pointee, so a table no pointer below the root targets is a node
@@ -4558,6 +4595,74 @@ tables-wire-fuzz-message-text-leg-negative-control: build/conformance-harness
 	@grep -m1 "FAILED" $(MESSAGE_TEXT_LEG_NC)/log
 	@echo "negative control: an emitted reader that accepts ill-formed message text turns the pinned vector RED"
 
+# ILL-FORMED TEXT IN A *string BLOB RECORD ON A MESSAGE BODY
+# (docs/SPEC-TABLES.md §3, §3.1, §3.3; schema#632), and the two vectors
+# message_blob_ill_formed_text and message_blob_zero_byte are what hold the two
+# engines to one answer on it. The blade in
+# tables-message-form-blob-negative-control holds each engine to the PAGE;
+# these two hold the engines to EACH OTHER, which is a different question and
+# the one a differential fuzzer exists to ask: before the repair both readers
+# placed the record, agreeing and agreeing wrongly, so nothing here could have
+# gone red. Each control repairs one engine and takes the rule back out of the
+# other, and the run must go red ON THE VECTOR.
+#
+# THE ASSERTION IS THE VECTOR REPLAYED ALONE, not a corpus pass, for the reason
+# the node-type control names: an enumerated mutant reaches the same check and
+# would name itself instead of the property.
+MESSAGE_BLOB_VECTOR := testdata/wire/tables/fuzz-vectors/message_blob_ill_formed_text.bin
+MESSAGE_BLOB_ZERO_VECTOR := testdata/wire/tables/fuzz-vectors/message_blob_zero_byte.bin
+
+MESSAGE_BLOB_ORACLE_NC := build/wire-fuzz-nc-message-blob-oracle
+.PHONY: tables-wire-fuzz-message-blob-oracle-negative-control
+tables-wire-fuzz-message-blob-oracle-negative-control: build/conformance-harness build/wire-fuzz-cpp
+	@rm -rf $(MESSAGE_BLOB_ORACLE_NC) && mkdir -p $(MESSAGE_BLOB_ORACLE_NC)
+	@go run ./tools/sabotage -name message-blob-accepts-ill-formed \
+		-out $(MESSAGE_BLOB_ORACLE_NC)/messagedecode.go.txt internal/tablewire/messagedecode.go
+	@printf '{"Replace":{"%s/internal/tablewire/messagedecode.go":"%s/$(MESSAGE_BLOB_ORACLE_NC)/messagedecode.go.txt"}}\n' \
+		"$(CURDIR)" "$(CURDIR)" > $(MESSAGE_BLOB_ORACLE_NC)/overlay.json
+	go build -overlay $(MESSAGE_BLOB_ORACLE_NC)/overlay.json -o $(MESSAGE_BLOB_ORACLE_NC)/harness ./test/conformance/harness
+	@for vector in $(MESSAGE_BLOB_VECTOR) $(MESSAGE_BLOB_ZERO_VECTOR); do \
+		if $(MESSAGE_BLOB_ORACLE_NC)/harness wire-fuzz --driver ./build/wire-fuzz-cpp \
+				--replay $$vector --unit blobdemo --root Catalog --message \
+				--failed $(MESSAGE_BLOB_ORACLE_NC)/failed.bin > $(MESSAGE_BLOB_ORACLE_NC)/log 2>&1; then \
+			echo "NEGATIVE CONTROL FAILED: the oracle places an ill-formed *string blob again and $$vector stayed green"; \
+			cat $(MESSAGE_BLOB_ORACLE_NC)/log; exit 1; \
+		fi; \
+		grep -q "$$(basename $$vector)" $(MESSAGE_BLOB_ORACLE_NC)/log || \
+			{ echo "NEGATIVE CONTROL FAILED: the wire fuzzer went red, but not on $$vector"; \
+			  cat $(MESSAGE_BLOB_ORACLE_NC)/log; exit 1; }; \
+		grep -m1 "FAILED" $(MESSAGE_BLOB_ORACLE_NC)/log; \
+	done
+	@echo "negative control: an oracle that places an ill-formed *string blob turns both pinned vectors RED"
+
+MESSAGE_BLOB_LEG_NC := build/wire-fuzz-nc-message-blob-leg
+.PHONY: tables-wire-fuzz-message-blob-leg-negative-control
+tables-wire-fuzz-message-blob-leg-negative-control: build/conformance-harness
+	@rm -rf $(MESSAGE_BLOB_LEG_NC) && mkdir -p $(MESSAGE_BLOB_LEG_NC)
+	@go run ./tools/sabotage -name message-emitter-blob-accepts-ill-formed \
+		-out $(MESSAGE_BLOB_LEG_NC)/messagevariable.go.txt internal/codegen/cpptable/messagevariable.go
+	@printf '{"Replace":{"%s/internal/codegen/cpptable/messagevariable.go":"%s/$(MESSAGE_BLOB_LEG_NC)/messagevariable.go.txt"}}\n' \
+		"$(CURDIR)" "$(CURDIR)" > $(MESSAGE_BLOB_LEG_NC)/overlay.json
+	go build -overlay $(MESSAGE_BLOB_LEG_NC)/overlay.json -o $(MESSAGE_BLOB_LEG_NC)/schema ./cmd/schema
+	$(call tables_generate,./$(MESSAGE_BLOB_LEG_NC)/schema,$(MESSAGE_BLOB_LEG_NC)/generated)
+	$(CXX) $(TABLES_CXXFLAGS) -O1 $(call tables_includes,$(MESSAGE_BLOB_LEG_NC)/generated) \
+		test/tables/wire_fuzz_main.cpp \
+		$(subst build/tables-generated/,$(MESSAGE_BLOB_LEG_NC)/generated/,$(CONFORMANCE_SOURCES)) \
+		-o $(MESSAGE_BLOB_LEG_NC)/leg
+	@for vector in $(MESSAGE_BLOB_VECTOR) $(MESSAGE_BLOB_ZERO_VECTOR); do \
+		if ./build/conformance-harness wire-fuzz --driver $(MESSAGE_BLOB_LEG_NC)/leg \
+				--replay $$vector --unit blobdemo --root Catalog --message \
+				--failed $(MESSAGE_BLOB_LEG_NC)/failed.bin > $(MESSAGE_BLOB_LEG_NC)/log 2>&1; then \
+			echo "NEGATIVE CONTROL FAILED: the emitted reader places an ill-formed *string blob again and $$vector stayed green"; \
+			cat $(MESSAGE_BLOB_LEG_NC)/log; exit 1; \
+		fi; \
+		grep -q "$$(basename $$vector)" $(MESSAGE_BLOB_LEG_NC)/log || \
+			{ echo "NEGATIVE CONTROL FAILED: the wire fuzzer went red, but not on $$vector"; \
+			  cat $(MESSAGE_BLOB_LEG_NC)/log; exit 1; }; \
+		grep -m1 "FAILED" $(MESSAGE_BLOB_LEG_NC)/log; \
+	done
+	@echo "negative control: an emitted reader that places an ill-formed *string blob turns both pinned vectors RED"
+
 # The GENERATED half of the data: the JSON text of every instance and the read
 # report of every evolution case, both from the compiler's own engine.
 .PHONY: conformance-generate

docs/SPEC-TABLES.md

@@ -5435,6 +5435,15 @@ entries announces about 5 KB once.
   lands whole and counts nothing. Red if a leg stores text the file form
   refuses, cuts a clamp inside a code point, keeps fewer bytes than the bound
   admits, or counts `clamped` on a payload that fits.
+- **The same content rule met at a NODE.** A `*string` blob record on a
+  form-`2` body carrying a truncated sequence, one carrying a zero byte, one
+  carrying an overlong encoding and one carrying a lead byte UTF-8 never
+  spells, each damage and terminal for the batch on §3.1's own terms. Beside
+  them a well-formed blob, which loads with a silent report, and the same
+  ill-formed bytes under the reserved `bytes` id, which loads with a silent
+  report too, because a `*bytes` blob is bytes and never text. Red if a leg
+  places a record the file form refuses, refuses a record the file form places,
+  or reads a `*bytes` blob as text.
 - **The pad, and what follows it.** A batch whose trailing bits to the byte
   boundary are not zero, and a buffer carrying a whole batch and then a byte
   more. Red if a leg reads either clean.

internal/codegen/cpptable/messagevariable.go

@@ -242,6 +242,17 @@ func (g *tableGen) emitVariableMessageSurface(st *ir.Struct) {
 	g.pf("        if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId )\n        {\n")
 	g.pf("            uint64_t length = 0;\n")
 	g.pf("            if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; }\n")
+	if g.rootReachesStringBlob(st) {
+		// A TEXT BLOB'S CONTENT IS REFUSED ON THE SAME TERMS as a kind 12
+		// payload (docs/SPEC-TABLES.md §3.1), through the runtime's own
+		// TableUtf8Valid, which is what the FILE form reads one with in
+		// pointers.go. The align above already left the bytes on a byte
+		// boundary, so the span goes to the check as it goes to the memcpy
+		// below. What differs from the file form is only the RECOVERY, which
+		// a bit stream does not have: the damage is TERMINAL for the batch,
+		// one malformed counts, and the bodies before it stand (§3.3).
+		g.pf("            if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; }\n")
+	}
 	g.pf("            if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); }\n")
 	g.pf("            r.offset += (int64_t) length * 8;\n")
 	g.pf("            continue;\n        }\n")

internal/tablewire/messagedecode.go

@@ -360,6 +360,16 @@ func (d *bitDecoder) nodeTable(inst *tabletext.Instance, st *decodeState) bool {
 				d.report.Unknown++
 				continue
 			}
+			// A TEXT BLOB'S CONTENT IS REFUSED ON THE SAME TERMS as a kind 12
+			// payload (§3.1), through the same textValid decodenodes.go reads
+			// the FILE form's record with, so there is one rule and no second
+			// copy. What differs is only the RECOVERY, which a bit stream does
+			// not have: the damage is TERMINAL for the batch, one malformed
+			// counts, and the bodies before it stand (§3.3).
+			if kind == ir.TString && !textValid(rec.blob) {
+				d.report.Malformed = true
+				return false
+			}
 			st.nodes[i] = Node{Blob: &tabletext.Blob{Data: rec.blob}, Kind: kind}
 			continue
 		}

test/conformance/harness/messagerules_test.go

@@ -1512,3 +1512,81 @@ func TestTheMessageFormsTextContentRuleAndClamp(t *testing.T) {
 		t.Errorf("a payload at the bound lands whole: kept %d bytes, %q", len(got), got)
 	}
 }
+
+// TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule: kinds `12` and
+// `33`'s content rule MET AT A NODE (docs/SPEC-TABLES.md §3.1) over a form-`2`
+// body (§3.3). §3.1 refuses a text blob's CONTENT on the file form's own
+// terms, and §3.3 says a form-`2` body's content rules are §3's, unchanged in
+// what they reject: a `*string` blob whose bytes are not well-formed UTF-8, or
+// which carries a zero byte, is DAMAGE and not data. What differs is only the
+// recovery, which a bit stream does not have, so the damage is TERMINAL for
+// the batch. Red if the message path places a blob the file path refuses.
+//
+// The instrument is `blobdemo`'s `Catalog`, whose numbering reaches a
+// `*string` blob through `note` and a `*bytes` blob through `thumb`. A blob
+// record is its own framing wherever it appears: the type reference, a length
+// at thirty-two raw bits, the ALIGN, then the bytes verbatim. A record is
+// numbered and placed whether or not a slot names it, so the rule is reached
+// by the record alone.
+func TestAStringBlobRecordOnAMessageBodyCarriesTheContentRule(t *testing.T) {
+	_, _, u := corpus(t)
+	unit, err := u.get("blobdemo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	model := tabletext.NewModel(unit)
+	v := vocabularyOf(t, unit)
+	// THE RESERVED IDS RIDE IN THE ANNOUNCEMENT'S TAIL whether or not a root
+	// names them (§3.1, §3.3), each a kind-0 entry that frames nothing
+	node := slotOf(t, v, ir.TableNodeWireId, 0)
+	stringType := slotOf(t, v, ir.StringWireTypeId, 0)
+	bytesType := slotOf(t, v, ir.BytesWireTypeId, 0)
+	body := func(typeSlot uint64, data []byte) []byte {
+		w := &bitw{}
+		w.put(node, v.RefBits())
+		w.put(1, 32)
+		w.put(typeSlot, v.RefBits())
+		w.put(uint64(len(data)), 32)
+		w.align()
+		w.bytes(data)
+		w.put(0, v.RefBits())
+		return batchOf(w)
+	}
+
+	// A WELL-FORMED BLOB IS ORDINARY, and the row is here first so a gate that
+	// went red by refusing everything is not one that holds the rule
+	_, ok, report, err := decodeOne(t, model, "Catalog", v, body(stringType, []byte("abcdefgh")))
+	if err != nil || !ok || !report.Silent() {
+		t.Fatalf("a well-formed *string blob on a message body loads silently: ok=%v err=%v report=%+v", ok, err, report)
+	}
+
+	// ILL-FORMED CONTENT IS DAMAGE AND IT IS TERMINAL: a truncated sequence, a
+	// zero byte, an overlong encoding and a lead byte UTF-8 never spells are
+	// each a payload that is not text at whatever length it arrived at
+	damaged := []struct {
+		name string
+		data []byte
+	}{
+		{"a truncated sequence", []byte{'p', 'a', 'c', 'k', 0xC3}},
+		{"a zero byte among the bytes", []byte{'p', 'a', 0x00, 'c', 'k'}},
+		{"an overlong encoding", []byte{'p', 0xC0, 0x80, 'k'}},
+		{"a lead byte UTF-8 never spells", []byte{'a', 'b', 'c', 0xFF, 'e', 'f', 'g', 'h'}},
+	}
+	for _, dr := range damaged {
+		_, loaded, rep, derr := decodeOne(t, model, "Catalog", v, body(stringType, dr.data))
+		if derr != nil {
+			t.Fatalf("%s: the decode errored: %v", dr.name, derr)
+		}
+		if loaded || !rep.Malformed || rep.Refused {
+			t.Errorf("%s in a *string blob record is damage, terminal for the batch (§3.1, §3.3): ok=%v report=%+v", dr.name, loaded, rep)
+		}
+	}
+
+	// AND A *bytes BLOB HAS NO SUCH RULE, because it is bytes and never text
+	// (§3.1): the same bytes under the reserved `bytes` id load clean, which
+	// holds the content rule to the `string` id alone
+	_, ok, report, err = decodeOne(t, model, "Catalog", v, body(bytesType, []byte{'p', 'a', 0x00, 'c', 0xFF}))
+	if err != nil || !ok || !report.Silent() {
+		t.Errorf("the same bytes under the reserved bytes id load silently (§3.1): ok=%v err=%v report=%+v", ok, err, report)
+	}
+}