tables: the message form's LoadRetain, and the manifest's retain surface (#525)

Makefile

@@ -3100,6 +3100,7 @@ test: toolchain build/schema_test build/schema_test_guard build/schema_test_tabl
 	$(MAKE) tables-retain
 	$(MAKE) tables-retain-fixed-class-negative-control
 	$(MAKE) tables-retain-message-form-negative-control
+	$(MAKE) tables-message-form-retain-negative-control
 	$(MAKE) tables-json-walk
 	$(MAKE) tables-json-graph-walk
 	$(MAKE) tables-json-negative-control
@@ -3542,6 +3543,43 @@ tables-retain-message-form-negative-control: build/tables-generated/.stamp
 	@grep -q "Retention writing the MESSAGE form is refused by name" build/retain-message-form.log || { echo "RETAIN GATE FAILED: the message-form refusal was not by name"; cat build/retain-message-form.log; exit 1; }
 	@echo "retention writing form 2 refuses BY NAME (docs/SPEC-TABLES.md §3.3)"
 
+# AND THE FORM 2 READ, WHICH RETAINS (docs/SPEC-TABLES.md §3.3, §6.6). The
+# unknown arm of a retaining message body SKIPS the entry and then re-reads the
+# bits it delimited, which is the whole of the form-2 capture. Take the second
+# half away and the skip is all that is left: the read is unchanged to the byte,
+# the six counters stand, and nothing about the batch says a field was lost.
+# What goes red is the pinned batch's own row of the retain gate, `retained` at
+# zero where the page says ten, which is what makes that row an instrument
+# rather than a restatement of what the emitter happens to do.
+#
+# The sabotage is one line of the emitter, through `go build -overlay`, so no
+# tracked file moves; the RT set is regenerated from the sabotaged compiler and
+# the gate is built against it.
+RETAIN_MSG_NC := build/retain-nc-message
+.PHONY: tables-message-form-retain-negative-control
+tables-message-form-retain-negative-control: bin/schema test/tables/retain_main.cpp
+	@rm -rf $(RETAIN_MSG_NC) && mkdir -p $(RETAIN_MSG_NC)
+	@go run ./tools/sabotage -name message-retain-no-capture \
+		-out $(RETAIN_MSG_NC)/messageload.gotext internal/codegen/cpptable/messageload.go
+	@printf '{"Replace":{"%s/internal/codegen/cpptable/messageload.go":"%s/$(RETAIN_MSG_NC)/messageload.gotext"}}\n' \
+		"$(CURDIR)" "$(CURDIR)" > $(RETAIN_MSG_NC)/overlay.json
+	go build -overlay $(RETAIN_MSG_NC)/overlay.json -o $(RETAIN_MSG_NC)/schema ./cmd/schema
+	@for u in rt1 rt2 rt3; do mkdir -p $(RETAIN_MSG_NC)/$$u; done
+	./$(RETAIN_MSG_NC)/schema generate --lang cpp --out $(RETAIN_MSG_NC)/rt1 test/tables/RT1.schema
+	./$(RETAIN_MSG_NC)/schema generate --lang cpp --out $(RETAIN_MSG_NC)/rt2 test/tables/RT2.schema
+	./$(RETAIN_MSG_NC)/schema generate --lang cpp --out $(RETAIN_MSG_NC)/rt3 test/tables/RT3.schema
+	$(CXX) $(TABLES_CXXFLAGS) -I$(RETAIN_MSG_NC)/rt1 -I$(RETAIN_MSG_NC)/rt2 -I$(RETAIN_MSG_NC)/rt3 \
+		-Itest/tables -I$(SERIALIZE) test/tables/retain_main.cpp -o $(RETAIN_MSG_NC)/gate
+	@if ./$(RETAIN_MSG_NC)/gate > $(RETAIN_MSG_NC)/log 2>&1; then \
+		echo "NEGATIVE CONTROL FAILED: the message path never entered the walk and the retain gate stayed green"; \
+		cat $(RETAIN_MSG_NC)/log; exit 1; \
+	fi
+	@grep -q "report.retained == 10" $(RETAIN_MSG_NC)/log || \
+		{ echo "NEGATIVE CONTROL FAILED: the gate went red, but not on the message batch's own row"; \
+		  cat $(RETAIN_MSG_NC)/log; exit 1; }
+	@grep -m1 "FAIL" $(RETAIN_MSG_NC)/log
+	@echo "negative control: removing the message path's call into the walk turns the pinned batch RED"
+
 # ---- A FLOAT RIDES AS ITS BIT PATTERN (docs/SPEC-TABLES.md §3, §4, SPEC.md
 # ---- §4.3, schema#480) -----------------------------------------------------
 #
@@ -4274,7 +4312,7 @@ CONFORMANCE_INCLUDES := -Ibuild/tables-generated/examples -Ibuild/tables-generat
 	-Ibuild/tables-generated/v2 -Ibuild/tables-generated/p1 -Ibuild/tables-generated/p3 \
 	-Ibuild/tables-generated/block -Ibuild/tables-generated/pointers \
 	-Ibuild/tables-generated/p2 -Ibuild/tables-generated/messages -Ibuild/tables-generated/stream \
-	-Ibuild/tables-generated/m1 -Ibuild/tables-generated/m2 -Ibuild/tables-generated/a1 -Ibuild/tables-generated/a2 -Ibuild/tables-generated/g1 -Ibuild/tables-generated/k1 -Ibuild/tables-generated/k2 -Ibuild/tables-generated/w1 -Ibuild/tables-generated/w2 -Ibuild/tables-generated/r1 -Ibuild/tables-generated/r2 -Ibuild/tables-generated/blobs -Itest/tables -Ibuild/tables-generated/scalars -Ibuild/tables-generated/scalars2 -Ibuild/tables-generated/backend -Ibuild/tables-generated/vocab -Ibuild/tables-generated/vocab9 -Ibuild/tables-generated/arms -Ibuild/tables-generated/wide -I$(SERIALIZE)
+	-Ibuild/tables-generated/m1 -Ibuild/tables-generated/m2 -Ibuild/tables-generated/a1 -Ibuild/tables-generated/a2 -Ibuild/tables-generated/g1 -Ibuild/tables-generated/k1 -Ibuild/tables-generated/k2 -Ibuild/tables-generated/w1 -Ibuild/tables-generated/w2 -Ibuild/tables-generated/r1 -Ibuild/tables-generated/r2 -Ibuild/tables-generated/blobs -Itest/tables -Ibuild/tables-generated/scalars -Ibuild/tables-generated/scalars2 -Ibuild/tables-generated/backend -Ibuild/tables-generated/vocab -Ibuild/tables-generated/vocab9 -Ibuild/tables-generated/arms -Ibuild/tables-generated/rt1 -Ibuild/tables-generated/wide -I$(SERIALIZE)
 CONFORMANCE_SOURCES = build/tables-generated/examples/TablesTable.cpp \
 	build/tables-generated/w1/W1Table.cpp build/tables-generated/w2/W2Table.cpp \
 	build/tables-generated/r1/R1Table.cpp build/tables-generated/r2/R2Table.cpp \

ROADMAP.md

@@ -64,12 +64,12 @@
 | unbounded arrays | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | the message form | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | bitpacked table message form | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| retain-unknown | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+| retain-unknown | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | doc comments and tags in the descriptors | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | the unit registry, UnitView | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | widening on read, and the refusal reasons | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 
-339 of 1180 cells are done. That is a count of features, not of effort, since the
+340 of 1180 cells are done. That is a count of features, not of effort, since the
 cells are not equal in size. The table will be printed as it stands with every
 release from here.
 

docs/SPEC-TABLES.md

@@ -4876,12 +4876,21 @@ damage (§3). A wire that had shipped would have taken `3` on that same rule.
 #### Retention, and what does not move
 
 **RETENTION (§6.6) ON A MESSAGE BODY: the load side is unchanged and the save
-side REFUSES.** The C++ reference carries the WRITE half of this
-paragraph, which is the refusal, and the form 2 `LoadRetain` is not built (§6.6).
+side REFUSES.** The C++ reference and the compiler's own engine carry both
+halves of this paragraph.
 
 `LoadRetain` reads a form-`2` body as it reads a file's, the resolving walk
 replacing every reference with the id it names, against the connection's
-vocabulary instead of a trailer. **`SaveRetain` writing form `2` REFUSES BY NAME
+vocabulary instead of a trailer. **THE RECORD IS THE FILE FORM'S OWN**, because
+a retained record carries the field's bytes with every reference resolved so
+that re-emitting it into any id table is correct, and the table it is re-emitted
+into is a FILE's: a bitpacked value is read at the width its announced shape
+states and written at the width the file form spells, which is this form's
+third difference (above) taken in the one direction retention has. **A BATCH
+TAKES ONE REGION AND ONE RETENTION BUFFER A BODY**, because each body carries
+its own node directory inside that one region (above), so a record's first step
+stays an index into the directory of the body it came from and `SaveRetain`'s
+accounting is the file form's unchanged. **`SaveRetain` writing form `2` REFUSES BY NAME
 and returns `-1`.** A form-`2` writer names entries through slots of a
 vocabulary the compiler settled, and a retained id is by definition one this
 build's closure does not contain, so it has no slot AND no announced shape, which
@@ -4995,6 +5004,11 @@ precedent.
   single message is the batch of one, and no singular verb is carried beside
   them: a surface with both would let a caller write one message a call and
   never learn that the batch is where the bandwidth is.
+- **`LoadRetainMessages` beside them**, the form-`2` read with retention on
+  (§6.6), PLURAL for the same reason and claimed on §11's own rule because it
+  is emitted. It takes one retention buffer a BODY, parallel to the caller's
+  array of roots, and there is no measure and no save beside it: retention
+  writing form `2` is `SaveRetainMessages`, which refuses by name (above).
 - **The refusal reason values `no_vocabulary`, `second_announcement`,
   `vocabulary_too_large`, `batch_too_large` and `message_form_as_file`** beside
   the form byte's own `newer_form`. `vocabulary_too_large` covers both bounds,
@@ -7438,9 +7452,15 @@ instance, and the discipline is to retain, edit values, and save, or to reload
 after a shape edit. The safety check is still read after `Save`, and it
 catches the drop.
 
-**HELD BY TEST, when it lands.** The rows the conformance manifest owes, each
-red for one reason, and **every row on a POINTERED unit** (the variable class,
-above), the fixed class's own row excepted:
+**HELD BY TEST.** The rows below, each red for one reason, and **every row on a
+POINTERED unit** (the variable class, above), the fixed class's own row
+excepted. **NINE OF THEM ARE THE CONFORMANCE MANIFEST'S OWN DATA**, on its
+`retain` and `retain-message` lines: the round trip at depth, the truncated
+record, the id list one short, the five excluded classes a wire can carry to the
+unknown arm, and the message form's tail. The rest are the two engines' own
+gates, because each asks something a shared row cannot: a record's BYTE cost is
+the port's own, an allocation audit is a language's own instrument, and a
+refusal by name is a compile error rather than an answer a driver writes.
 
 - a wire whose unknown fields sit at three depths, retained and re-emitted,
   the save pinned as a byte string of its own. Red if a field is lost,
@@ -7530,11 +7550,10 @@ negative controls, one per engine, stand beside the fuzzer's (§4.2).
 
 **Backend status: the C++ REFERENCE and the ORACLE carry it, and no port
 does.** The reference emits `TableRetain`, the three verbs on every
-variable-class root, the refusal on every fixed-class one, and a second family
-of body functions beside the three the wire already had, so `Load`, `Measure`
-and `Save` are unchanged. What is still owed is the eight ports and the MESSAGE
-form's `LoadRetain` (§3.3): the form 2 write refuses by name and the form 2
-read is not built.
+variable-class root, the refusal on every fixed-class one, the MESSAGE form's
+own `LoadRetainMessages` beside them (§3.3), and a second family of body
+functions beside the three the wire already had, so `Load`, `Measure` and
+`Save` are unchanged. What is still owed is the eight ports.
 
 ## 7. The cooked form
 
@@ -10139,9 +10158,9 @@ in build version (§20.5).
   types share one symbol table (§13.1), which is what makes the generated
   surface unprefixed and collision-free — so every name a closure member
   claims is refused to everything else. A member `X` claims `X` followed by
-  each of these **52 suffixes**, and a declaration spelling one of them is
+  each of these **53 suffixes**, and a declaration spelling one of them is
   refused naming the collision — the block form's nine and the C backend's
-  seven follow below, for **68 in all**:
+  seven follow below, for **69 in all**:
 
   ```
   Measure  MeasureBody  Save  SaveBody  SaveBodyFields  Load  LoadBody
@@ -10153,27 +10172,29 @@ in build version (§20.5).
   Open  TableFields  TableInfo
   FromJson  ToJson  ToJsonMeasure  Table
   MeasureMessages  SaveMessages  LoadMessages
-  LoadRetain  MeasureRetain  SaveRetain  SaveRetainMessages
+  LoadRetain  MeasureRetain  SaveRetain  LoadRetainMessages  SaveRetainMessages
   LoadBodyRetain  MeasureBodyRetain  SaveBodyRetain  SaveBodyFieldsRetain
   MeasureWireRetain  SaveWireRetain  NodeBodyRetain
   ```
 
   The set is claimed for EVERY closure member, not only pointer-bearing
   ones: a table gains or loses pointers as an edit, and a name that was
   free yesterday must not become a collision tomorrow. That list is the
-  checker's own, and this section is held to it: the three lists here, 52, then
+  checker's own, and this section is held to it: the three lists here, 53, then
   the block form's nine, then the C backend's seven, are `tableGeneratedVerbs`
-  entire, spelling for spelling and 68 in all, because a claim the page states
+  entire, spelling for spelling and 69 in all, because a claim the page states
   and the checker does not make is a name a user may take.
 
-  **RETAIN-UNKNOWN'S ELEVEN ARE THREE AND EIGHT** (§6.6). `LoadRetain`,
+  **RETAIN-UNKNOWN'S TWELVE ARE THREE AND NINE** (§6.6). `LoadRetain`,
   `MeasureRetain` and `SaveRetain` are the SURFACE the feature owes this
-  section. The other eight are what carries them, and they are claimed on
+  section. The other nine are what carries them, and they are claimed on
   this list's own rule because they are EMITTED: the second family of body
   functions, the wire pair a pointered root takes, the node dispatch that
-  reaches each record's own body, and `SaveRetainMessages`, which is REFUSED
-  BY NAME on a form 2 write (§3.3) and is a definition rather than only a
-  claim for exactly that reason.
+  reaches each record's own body, and the MESSAGE form's two.
+  `LoadRetainMessages` is the form 2 READ, which resolves against the
+  connection's vocabulary where the file form resolves against a trailer, and
+  `SaveRetainMessages` is the form 2 WRITE, REFUSED BY NAME (§3.3) and a
+  definition rather than only a claim for exactly that reason.
 
   **`Open` AND `Cook` ARE BOTH EMITTED NOW — in different languages, and that is
   what the C# rule below is for. `OpenWalk` was RETIRED.** The C++ table backend

docs/VERSIONING.md

@@ -1071,10 +1071,10 @@ repository not yet behind it. The 3.0.0 release holds the list at zero.
 - #432: the cook triple, and the byte-order sentences in five places.
 - #441: the retired-names ledger.
 - #446: the evolution table's fixtures.
-- #525: retain-unknown in the eight ports, the MESSAGE form's `LoadRetain`,
-  and the conformance rows. The C++ reference, the two report counters,
-  `internal/tablewire`'s own retention and the wire fuzzer's retention leg,
-  which needed it, are built.
+- #525: retain-unknown in the eight ports. The C++ reference, the two report
+  counters, `internal/tablewire`'s own retention, the wire fuzzer's retention
+  leg, which needed it, the MESSAGE form's `LoadRetain` and the conformance
+  manifest's retain rows are built.
 - #522: `*wstring`, the unbounded twin of wide text — the blob record is
   specified and no backend emits one (SPEC-TABLES.md §2.5). The BOUNDED
   spelling's table half, kind `33`, has landed in the C++ reference and the

internal/check/check.go

@@ -3641,9 +3641,11 @@ var tableGeneratedVerbs = []string{
 	// SaveRetain, and the rest are what carries them: the second family of
 	// body functions,
 	// the wire pair over a pointered root, the node dispatch that reaches each
-	// record's body, and the form 2 write, which is refused by name (§3.3).
+	// record's body, and the MESSAGE form's two: the form 2 READ, which
+	// retains against the connection's vocabulary, and the form 2 WRITE, which
+	// is refused by name (§3.3).
 	// Every one is emitted, so every one is claimed on this list's own rule.
-	"LoadRetain", "MeasureRetain", "SaveRetain", "SaveRetainMessages",
+	"LoadRetain", "MeasureRetain", "SaveRetain", "LoadRetainMessages", "SaveRetainMessages",
 	"LoadBodyRetain", "MeasureBodyRetain", "SaveBodyRetain", "SaveBodyFieldsRetain",
 	"MeasureWireRetain", "SaveWireRetain", "NodeBodyRetain",
 	"FromJson", "ToJson", "ToJsonMeasure",

internal/codegen/cpptable/cpptable.go

@@ -146,6 +146,12 @@ type tableGen struct {
 	// unionField is the union FIELD whose arms are being emitted, which is
 	// what an arm's own descent takes its ordinal from.
 	unionField *ir.Field
+	// retainGate is the condition under which a nested MESSAGE read captures
+	// into the caller's store at all, and empty where it always does. A bit
+	// stream has to be walked past, so an element this build is dropping is
+	// decoded into a scratch where a file's reader steps over it by its
+	// length, and nothing in that body is a field of this region (§6.6, §3.3).
+	retainGate string
 }
 
 // step renders one path step at a child-body descent: the field's ordinal in

internal/codegen/cpptable/messagecodec.go

@@ -53,13 +53,47 @@ func (g *tableGen) msgSaveCall(name, expr string) string {
 	return fmt.Sprintf("%sSaveMessageBody( w, %s )", name, expr)
 }
 
-func (g *tableGen) msgLoadCall(name, reader, expr string) string {
+func (g *tableGen) msgLoadCall(f *ir.Field, name, reader, expr string) string {
+	if g.retain {
+		// THE PATH IS THREADED exactly as the file form threads it
+		// (docs/SPEC-TABLES.md §6.6): the step is computed LOCALLY, at the
+		// moment the walk descends, and the store is NULL where the element is
+		// one the reader is dropping.
+		if g.isVar(name) {
+			return fmt.Sprintf("%sLoadMessageBodyRetain( %s, vocabulary, report, nodes, index_bits, %s, %s, %s )",
+				name, reader, expr, g.retainStore(), g.step(f))
+		}
+		return fmt.Sprintf("%sLoadMessageBodyRetain( %s, vocabulary, report, index_bits, %s, %s, %s )",
+			name, reader, expr, g.retainStore(), g.step(f))
+	}
 	if g.isVar(name) {
 		return fmt.Sprintf("%sLoadMessageBody( %s, vocabulary, report, nodes, index_bits, %s )", name, reader, expr)
 	}
 	return fmt.Sprintf("%sLoadMessageBody( %s, vocabulary, report, index_bits, %s )", name, reader, expr)
 }
 
+// retainStore is the caller's buffer as a NESTED READ takes it. A bit stream
+// has to be walked past, so an element this build is DROPPING, one past its
+// own array bound, one under a keyed key it cannot name, is decoded into a
+// scratch where a file's reader steps over it by its length. Nothing in that
+// body is a field of this region, so the store it captures into is none
+// (docs/SPEC-TABLES.md §6.6).
+func (g *tableGen) retainStore() string {
+	if g.retainGate == "" {
+		return "retain"
+	}
+	return fmt.Sprintf("( %s ? retain : NULL )", g.retainGate)
+}
+
+// inDrop runs `emit` with the store gated on the element being this build's,
+// and puts back what was there.
+func (g *tableGen) inDrop(gate string, emit func()) {
+	was := g.retainGate
+	g.retainGate = gate
+	emit()
+	g.retainGate = was
+}
+
 // msgEnter opens one nesting level of the message codec's emission and answers
 // the suffix its locals carry; msgLeave closes it. The outermost payload's
 // names are bare, and every level under it is numbered, so a decode inside a