diff --git a/Makefile b/Makefile index f9aea0f32..07e3c3af0 100644 --- a/Makefile +++ b/Makefile @@ -3100,6 +3100,7 @@ test: toolchain build/schema_test build/schema_test_guard build/schema_test_tabl $(MAKE) tables-retain $(MAKE) tables-retain-fixed-class-negative-control $(MAKE) tables-retain-message-form-negative-control + $(MAKE) tables-message-form-retain-negative-control $(MAKE) tables-json-walk $(MAKE) tables-json-graph-walk $(MAKE) tables-json-negative-control @@ -3542,6 +3543,43 @@ tables-retain-message-form-negative-control: build/tables-generated/.stamp @grep -q "Retention writing the MESSAGE form is refused by name" build/retain-message-form.log || { echo "RETAIN GATE FAILED: the message-form refusal was not by name"; cat build/retain-message-form.log; exit 1; } @echo "retention writing form 2 refuses BY NAME (docs/SPEC-TABLES.md §3.3)" +# AND THE FORM 2 READ, WHICH RETAINS (docs/SPEC-TABLES.md §3.3, §6.6). The +# unknown arm of a retaining message body SKIPS the entry and then re-reads the +# bits it delimited, which is the whole of the form-2 capture. Take the second +# half away and the skip is all that is left: the read is unchanged to the byte, +# the six counters stand, and nothing about the batch says a field was lost. +# What goes red is the pinned batch's own row of the retain gate, `retained` at +# zero where the page says ten, which is what makes that row an instrument +# rather than a restatement of what the emitter happens to do. +# +# The sabotage is one line of the emitter, through `go build -overlay`, so no +# tracked file moves; the RT set is regenerated from the sabotaged compiler and +# the gate is built against it. +RETAIN_MSG_NC := build/retain-nc-message +.PHONY: tables-message-form-retain-negative-control +tables-message-form-retain-negative-control: bin/schema test/tables/retain_main.cpp + @rm -rf $(RETAIN_MSG_NC) && mkdir -p $(RETAIN_MSG_NC) + @go run ./tools/sabotage -name message-retain-no-capture \ + -out $(RETAIN_MSG_NC)/messageload.gotext internal/codegen/cpptable/messageload.go + @printf '{"Replace":{"%s/internal/codegen/cpptable/messageload.go":"%s/$(RETAIN_MSG_NC)/messageload.gotext"}}\n' \ + "$(CURDIR)" "$(CURDIR)" > $(RETAIN_MSG_NC)/overlay.json + go build -overlay $(RETAIN_MSG_NC)/overlay.json -o $(RETAIN_MSG_NC)/schema ./cmd/schema + @for u in rt1 rt2 rt3; do mkdir -p $(RETAIN_MSG_NC)/$$u; done + ./$(RETAIN_MSG_NC)/schema generate --lang cpp --out $(RETAIN_MSG_NC)/rt1 test/tables/RT1.schema + ./$(RETAIN_MSG_NC)/schema generate --lang cpp --out $(RETAIN_MSG_NC)/rt2 test/tables/RT2.schema + ./$(RETAIN_MSG_NC)/schema generate --lang cpp --out $(RETAIN_MSG_NC)/rt3 test/tables/RT3.schema + $(CXX) $(TABLES_CXXFLAGS) -I$(RETAIN_MSG_NC)/rt1 -I$(RETAIN_MSG_NC)/rt2 -I$(RETAIN_MSG_NC)/rt3 \ + -Itest/tables -I$(SERIALIZE) test/tables/retain_main.cpp -o $(RETAIN_MSG_NC)/gate + @if ./$(RETAIN_MSG_NC)/gate > $(RETAIN_MSG_NC)/log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the message path never entered the walk and the retain gate stayed green"; \ + cat $(RETAIN_MSG_NC)/log; exit 1; \ + fi + @grep -q "report.retained == 10" $(RETAIN_MSG_NC)/log || \ + { echo "NEGATIVE CONTROL FAILED: the gate went red, but not on the message batch's own row"; \ + cat $(RETAIN_MSG_NC)/log; exit 1; } + @grep -m1 "FAIL" $(RETAIN_MSG_NC)/log + @echo "negative control: removing the message path's call into the walk turns the pinned batch RED" + # ---- A FLOAT RIDES AS ITS BIT PATTERN (docs/SPEC-TABLES.md §3, §4, SPEC.md # ---- §4.3, schema#480) ----------------------------------------------------- # @@ -4274,7 +4312,7 @@ CONFORMANCE_INCLUDES := -Ibuild/tables-generated/examples -Ibuild/tables-generat -Ibuild/tables-generated/v2 -Ibuild/tables-generated/p1 -Ibuild/tables-generated/p3 \ -Ibuild/tables-generated/block -Ibuild/tables-generated/pointers \ -Ibuild/tables-generated/p2 -Ibuild/tables-generated/messages -Ibuild/tables-generated/stream \ - -Ibuild/tables-generated/m1 -Ibuild/tables-generated/m2 -Ibuild/tables-generated/a1 -Ibuild/tables-generated/a2 -Ibuild/tables-generated/g1 -Ibuild/tables-generated/k1 -Ibuild/tables-generated/k2 -Ibuild/tables-generated/w1 -Ibuild/tables-generated/w2 -Ibuild/tables-generated/r1 -Ibuild/tables-generated/r2 -Ibuild/tables-generated/blobs -Itest/tables -Ibuild/tables-generated/scalars -Ibuild/tables-generated/scalars2 -Ibuild/tables-generated/backend -Ibuild/tables-generated/vocab -Ibuild/tables-generated/vocab9 -Ibuild/tables-generated/arms -Ibuild/tables-generated/wide -I$(SERIALIZE) + -Ibuild/tables-generated/m1 -Ibuild/tables-generated/m2 -Ibuild/tables-generated/a1 -Ibuild/tables-generated/a2 -Ibuild/tables-generated/g1 -Ibuild/tables-generated/k1 -Ibuild/tables-generated/k2 -Ibuild/tables-generated/w1 -Ibuild/tables-generated/w2 -Ibuild/tables-generated/r1 -Ibuild/tables-generated/r2 -Ibuild/tables-generated/blobs -Itest/tables -Ibuild/tables-generated/scalars -Ibuild/tables-generated/scalars2 -Ibuild/tables-generated/backend -Ibuild/tables-generated/vocab -Ibuild/tables-generated/vocab9 -Ibuild/tables-generated/arms -Ibuild/tables-generated/rt1 -Ibuild/tables-generated/wide -I$(SERIALIZE) CONFORMANCE_SOURCES = build/tables-generated/examples/TablesTable.cpp \ build/tables-generated/w1/W1Table.cpp build/tables-generated/w2/W2Table.cpp \ build/tables-generated/r1/R1Table.cpp build/tables-generated/r2/R2Table.cpp \ diff --git a/ROADMAP.md b/ROADMAP.md index d07f38438..a3a8d545e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -64,12 +64,12 @@ | unbounded arrays | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | the message form | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | bitpacked table message form | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| retain-unknown | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| retain-unknown | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | doc comments and tags in the descriptors | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | the unit registry, UnitView | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | widening on read, and the refusal reasons | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -339 of 1180 cells are done. That is a count of features, not of effort, since the +340 of 1180 cells are done. That is a count of features, not of effort, since the cells are not equal in size. The table will be printed as it stands with every release from here. diff --git a/docs/SPEC-TABLES.md b/docs/SPEC-TABLES.md index 4a057fc01..b1f2ee05a 100644 --- a/docs/SPEC-TABLES.md +++ b/docs/SPEC-TABLES.md @@ -4876,12 +4876,21 @@ damage (§3). A wire that had shipped would have taken `3` on that same rule. #### Retention, and what does not move **RETENTION (§6.6) ON A MESSAGE BODY: the load side is unchanged and the save -side REFUSES.** The C++ reference carries the WRITE half of this -paragraph, which is the refusal, and the form 2 `LoadRetain` is not built (§6.6). +side REFUSES.** The C++ reference and the compiler's own engine carry both +halves of this paragraph. `LoadRetain` reads a form-`2` body as it reads a file's, the resolving walk replacing every reference with the id it names, against the connection's -vocabulary instead of a trailer. **`SaveRetain` writing form `2` REFUSES BY NAME +vocabulary instead of a trailer. **THE RECORD IS THE FILE FORM'S OWN**, because +a retained record carries the field's bytes with every reference resolved so +that re-emitting it into any id table is correct, and the table it is re-emitted +into is a FILE's: a bitpacked value is read at the width its announced shape +states and written at the width the file form spells, which is this form's +third difference (above) taken in the one direction retention has. **A BATCH +TAKES ONE REGION AND ONE RETENTION BUFFER A BODY**, because each body carries +its own node directory inside that one region (above), so a record's first step +stays an index into the directory of the body it came from and `SaveRetain`'s +accounting is the file form's unchanged. **`SaveRetain` writing form `2` REFUSES BY NAME and returns `-1`.** A form-`2` writer names entries through slots of a vocabulary the compiler settled, and a retained id is by definition one this build's closure does not contain, so it has no slot AND no announced shape, which @@ -4995,6 +5004,11 @@ precedent. single message is the batch of one, and no singular verb is carried beside them: a surface with both would let a caller write one message a call and never learn that the batch is where the bandwidth is. +- **`LoadRetainMessages` beside them**, the form-`2` read with retention on + (§6.6), PLURAL for the same reason and claimed on §11's own rule because it + is emitted. It takes one retention buffer a BODY, parallel to the caller's + array of roots, and there is no measure and no save beside it: retention + writing form `2` is `SaveRetainMessages`, which refuses by name (above). - **The refusal reason values `no_vocabulary`, `second_announcement`, `vocabulary_too_large`, `batch_too_large` and `message_form_as_file`** beside the form byte's own `newer_form`. `vocabulary_too_large` covers both bounds, @@ -7438,9 +7452,15 @@ instance, and the discipline is to retain, edit values, and save, or to reload after a shape edit. The safety check is still read after `Save`, and it catches the drop. -**HELD BY TEST, when it lands.** The rows the conformance manifest owes, each -red for one reason, and **every row on a POINTERED unit** (the variable class, -above), the fixed class's own row excepted: +**HELD BY TEST.** The rows below, each red for one reason, and **every row on a +POINTERED unit** (the variable class, above), the fixed class's own row +excepted. **NINE OF THEM ARE THE CONFORMANCE MANIFEST'S OWN DATA**, on its +`retain` and `retain-message` lines: the round trip at depth, the truncated +record, the id list one short, the five excluded classes a wire can carry to the +unknown arm, and the message form's tail. The rest are the two engines' own +gates, because each asks something a shared row cannot: a record's BYTE cost is +the port's own, an allocation audit is a language's own instrument, and a +refusal by name is a compile error rather than an answer a driver writes. - a wire whose unknown fields sit at three depths, retained and re-emitted, the save pinned as a byte string of its own. Red if a field is lost, @@ -7530,11 +7550,10 @@ negative controls, one per engine, stand beside the fuzzer's (§4.2). **Backend status: the C++ REFERENCE and the ORACLE carry it, and no port does.** The reference emits `TableRetain`, the three verbs on every -variable-class root, the refusal on every fixed-class one, and a second family -of body functions beside the three the wire already had, so `Load`, `Measure` -and `Save` are unchanged. What is still owed is the eight ports and the MESSAGE -form's `LoadRetain` (§3.3): the form 2 write refuses by name and the form 2 -read is not built. +variable-class root, the refusal on every fixed-class one, the MESSAGE form's +own `LoadRetainMessages` beside them (§3.3), and a second family of body +functions beside the three the wire already had, so `Load`, `Measure` and +`Save` are unchanged. What is still owed is the eight ports. ## 7. The cooked form @@ -10139,9 +10158,9 @@ in build version (§20.5). types share one symbol table (§13.1), which is what makes the generated surface unprefixed and collision-free — so every name a closure member claims is refused to everything else. A member `X` claims `X` followed by - each of these **52 suffixes**, and a declaration spelling one of them is + each of these **53 suffixes**, and a declaration spelling one of them is refused naming the collision — the block form's nine and the C backend's - seven follow below, for **68 in all**: + seven follow below, for **69 in all**: ``` Measure MeasureBody Save SaveBody SaveBodyFields Load LoadBody @@ -10153,7 +10172,7 @@ in build version (§20.5). Open TableFields TableInfo FromJson ToJson ToJsonMeasure Table MeasureMessages SaveMessages LoadMessages - LoadRetain MeasureRetain SaveRetain SaveRetainMessages + LoadRetain MeasureRetain SaveRetain LoadRetainMessages SaveRetainMessages LoadBodyRetain MeasureBodyRetain SaveBodyRetain SaveBodyFieldsRetain MeasureWireRetain SaveWireRetain NodeBodyRetain ``` @@ -10161,19 +10180,21 @@ in build version (§20.5). The set is claimed for EVERY closure member, not only pointer-bearing ones: a table gains or loses pointers as an edit, and a name that was free yesterday must not become a collision tomorrow. That list is the - checker's own, and this section is held to it: the three lists here, 52, then + checker's own, and this section is held to it: the three lists here, 53, then the block form's nine, then the C backend's seven, are `tableGeneratedVerbs` - entire, spelling for spelling and 68 in all, because a claim the page states + entire, spelling for spelling and 69 in all, because a claim the page states and the checker does not make is a name a user may take. - **RETAIN-UNKNOWN'S ELEVEN ARE THREE AND EIGHT** (§6.6). `LoadRetain`, + **RETAIN-UNKNOWN'S TWELVE ARE THREE AND NINE** (§6.6). `LoadRetain`, `MeasureRetain` and `SaveRetain` are the SURFACE the feature owes this - section. The other eight are what carries them, and they are claimed on + section. The other nine are what carries them, and they are claimed on this list's own rule because they are EMITTED: the second family of body functions, the wire pair a pointered root takes, the node dispatch that - reaches each record's own body, and `SaveRetainMessages`, which is REFUSED - BY NAME on a form 2 write (§3.3) and is a definition rather than only a - claim for exactly that reason. + reaches each record's own body, and the MESSAGE form's two. + `LoadRetainMessages` is the form 2 READ, which resolves against the + connection's vocabulary where the file form resolves against a trailer, and + `SaveRetainMessages` is the form 2 WRITE, REFUSED BY NAME (§3.3) and a + definition rather than only a claim for exactly that reason. **`Open` AND `Cook` ARE BOTH EMITTED NOW — in different languages, and that is what the C# rule below is for. `OpenWalk` was RETIRED.** The C++ table backend diff --git a/docs/VERSIONING.md b/docs/VERSIONING.md index 07587efe6..3efb162df 100644 --- a/docs/VERSIONING.md +++ b/docs/VERSIONING.md @@ -1071,10 +1071,10 @@ repository not yet behind it. The 3.0.0 release holds the list at zero. - #432: the cook triple, and the byte-order sentences in five places. - #441: the retired-names ledger. - #446: the evolution table's fixtures. -- #525: retain-unknown in the eight ports, the MESSAGE form's `LoadRetain`, - and the conformance rows. The C++ reference, the two report counters, - `internal/tablewire`'s own retention and the wire fuzzer's retention leg, - which needed it, are built. +- #525: retain-unknown in the eight ports. The C++ reference, the two report + counters, `internal/tablewire`'s own retention, the wire fuzzer's retention + leg, which needed it, the MESSAGE form's `LoadRetain` and the conformance + manifest's retain rows are built. - #522: `*wstring`, the unbounded twin of wide text — the blob record is specified and no backend emits one (SPEC-TABLES.md §2.5). The BOUNDED spelling's table half, kind `33`, has landed in the C++ reference and the diff --git a/internal/check/check.go b/internal/check/check.go index 6c92f567a..2b94b1af9 100644 --- a/internal/check/check.go +++ b/internal/check/check.go @@ -3641,9 +3641,11 @@ var tableGeneratedVerbs = []string{ // SaveRetain, and the rest are what carries them: the second family of // body functions, // the wire pair over a pointered root, the node dispatch that reaches each - // record's body, and the form 2 write, which is refused by name (§3.3). + // record's body, and the MESSAGE form's two: the form 2 READ, which + // retains against the connection's vocabulary, and the form 2 WRITE, which + // is refused by name (§3.3). // Every one is emitted, so every one is claimed on this list's own rule. - "LoadRetain", "MeasureRetain", "SaveRetain", "SaveRetainMessages", + "LoadRetain", "MeasureRetain", "SaveRetain", "LoadRetainMessages", "SaveRetainMessages", "LoadBodyRetain", "MeasureBodyRetain", "SaveBodyRetain", "SaveBodyFieldsRetain", "MeasureWireRetain", "SaveWireRetain", "NodeBodyRetain", "FromJson", "ToJson", "ToJsonMeasure", diff --git a/internal/codegen/cpptable/cpptable.go b/internal/codegen/cpptable/cpptable.go index 4899905a7..5d7253647 100644 --- a/internal/codegen/cpptable/cpptable.go +++ b/internal/codegen/cpptable/cpptable.go @@ -146,6 +146,12 @@ type tableGen struct { // unionField is the union FIELD whose arms are being emitted, which is // what an arm's own descent takes its ordinal from. unionField *ir.Field + // retainGate is the condition under which a nested MESSAGE read captures + // into the caller's store at all, and empty where it always does. A bit + // stream has to be walked past, so an element this build is dropping is + // decoded into a scratch where a file's reader steps over it by its + // length, and nothing in that body is a field of this region (§6.6, §3.3). + retainGate string } // step renders one path step at a child-body descent: the field's ordinal in diff --git a/internal/codegen/cpptable/messagecodec.go b/internal/codegen/cpptable/messagecodec.go index be7d5c32d..83596b6a2 100644 --- a/internal/codegen/cpptable/messagecodec.go +++ b/internal/codegen/cpptable/messagecodec.go @@ -53,13 +53,47 @@ func (g *tableGen) msgSaveCall(name, expr string) string { return fmt.Sprintf("%sSaveMessageBody( w, %s )", name, expr) } -func (g *tableGen) msgLoadCall(name, reader, expr string) string { +func (g *tableGen) msgLoadCall(f *ir.Field, name, reader, expr string) string { + if g.retain { + // THE PATH IS THREADED exactly as the file form threads it + // (docs/SPEC-TABLES.md §6.6): the step is computed LOCALLY, at the + // moment the walk descends, and the store is NULL where the element is + // one the reader is dropping. + if g.isVar(name) { + return fmt.Sprintf("%sLoadMessageBodyRetain( %s, vocabulary, report, nodes, index_bits, %s, %s, %s )", + name, reader, expr, g.retainStore(), g.step(f)) + } + return fmt.Sprintf("%sLoadMessageBodyRetain( %s, vocabulary, report, index_bits, %s, %s, %s )", + name, reader, expr, g.retainStore(), g.step(f)) + } if g.isVar(name) { return fmt.Sprintf("%sLoadMessageBody( %s, vocabulary, report, nodes, index_bits, %s )", name, reader, expr) } return fmt.Sprintf("%sLoadMessageBody( %s, vocabulary, report, index_bits, %s )", name, reader, expr) } +// retainStore is the caller's buffer as a NESTED READ takes it. A bit stream +// has to be walked past, so an element this build is DROPPING, one past its +// own array bound, one under a keyed key it cannot name, is decoded into a +// scratch where a file's reader steps over it by its length. Nothing in that +// body is a field of this region, so the store it captures into is none +// (docs/SPEC-TABLES.md §6.6). +func (g *tableGen) retainStore() string { + if g.retainGate == "" { + return "retain" + } + return fmt.Sprintf("( %s ? retain : NULL )", g.retainGate) +} + +// inDrop runs `emit` with the store gated on the element being this build's, +// and puts back what was there. +func (g *tableGen) inDrop(gate string, emit func()) { + was := g.retainGate + g.retainGate = gate + emit() + g.retainGate = was +} + // msgEnter opens one nesting level of the message codec's emission and answers // the suffix its locals carry; msgLeave closes it. The outermost payload's // names are bare, and every level under it is numbered, so a decode inside a diff --git a/internal/codegen/cpptable/messageload.go b/internal/codegen/cpptable/messageload.go index 29b612c11..35b58f080 100644 --- a/internal/codegen/cpptable/messageload.go +++ b/internal/codegen/cpptable/messageload.go @@ -20,6 +20,7 @@ package cpptable import ( "fmt" + "strconv" "github.com/mas-bandwidth/schema/v2/ir" ) @@ -29,13 +30,28 @@ func (g *tableGen) emitMessageLoadBody(st *ir.Struct) { g.pf("// defaults first, then whatever the wire says, field by field. An entry this\n") g.pf("// build cannot name is skipped by its SHAPE and counted; one whose kind is\n") g.pf("// not this field's is a kind mismatch and skipped the same way.\n") + if g.retain { + g.pf("// UNDER RETENTION the same body keeps the fields this build cannot name,\n") + g.pf("// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the\n") + g.pf("// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6).\n") + g.pf("// The reader's own data is exactly what it would have been with retention off.\n") + } if g.isVar(st.Name) { - g.pf("inline bool %sLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, %s & value )\n{\n", st.Name, st.Name) + g.pf("inline bool %s( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, %s & value%s )\n{\n", + g.verb(st.Name, "LoadMessageBody"), st.Name, g.retainParams()) g.pf(" (void) nodes; (void) index_bits;\n") } else { - g.pf("inline bool %sLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, %s & value )\n{\n", st.Name, st.Name) + g.pf("inline bool %s( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, %s & value%s )\n{\n", + g.verb(st.Name, "LoadMessageBody"), st.Name, g.retainParams()) } g.pf(" %sReset( value );\n", st.Name) + if g.retain { + g.pf(" // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT\n") + g.pf(" // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever\n") + g.pf(" // an earlier occurrence of it left is discarded before the winning one is\n") + g.pf(" // read. The discard moves neither counter.\n") + g.pf(" TableRetainDiscardBody( retain, path );\n") + } g.pf(" for ( ;; )\n {\n") g.pf(" uint64_t ref = 0;\n") g.pf(" if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; }\n") @@ -64,6 +80,22 @@ func (g *tableGen) emitMessageLoadBody(st *ir.Struct) { } g.pf(" default:\n") g.pf(" report->unknown++;\n") + if g.retain { + // THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits + // (§3.3): the plain read's verdict on this entry is the SKIP's, + // whether or not this build retains, and the capture re-reads the bits + // the skip delimited. It drops the record instead where an excluded + // class, a full buffer or a shape the walk cannot frame says it cannot + // be kept, and the read continues either way (§6.6). + g.pf(" {\n") + g.pf(" const int64_t unknown_at = r.offset;\n") + g.pf(" if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; }\n") + g.pf(" TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at );\n") + g.pf(" }\n") + g.pf(" break;\n") + g.pf(" }\n }\n}\n\n") + return + } g.pf(" if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; }\n") g.pf(" break;\n") g.pf(" }\n }\n}\n\n") @@ -102,6 +134,15 @@ func (g *tableGen) emitMessageWidenBranch(f *ir.Field, mine ir.TableVocabularyEn func (g *tableGen) emitMessageReadField(f *ir.Field) { ind := " " name := f.Name + if g.retain && fieldDiscardsWhole(f) { + // THE FIELD IS BEING READ AGAIN (docs/SPEC-TABLES.md §6.6): a repeated + // table field and a union whose arm is written again both replace + // everything the earlier occurrence held, so every record under this + // field goes before the winning one is read. A map, a list and a + // bounded array take this at the point their own read commits to + // replace instead, and an enum-keyed array never takes it at all. + g.pf("%sTableRetainDiscardField( retain, path, %d );\n", ind, g.ordinal[f]) + } switch { case f.IsMap(): g.emitMessageReadMap(f, ind) @@ -121,7 +162,7 @@ func (g *tableGen) emitMessageReadField(f *ir.Field) { case tableScalarKind(f) == tkUnion: g.emitMessageReadUnion(f, "value."+name, ind) case tableScalarKind(f) == tkTable: - g.pf("%sif ( !%s ) { return false; }\n", ind, g.msgLoadCall(f.Type.Name, "r", "value."+name)) + g.pf("%sif ( !%s ) { return false; }\n", ind, g.msgLoadCall(f, f.Type.Name, "r", "value."+name)) case tableScalarKind(f) == tkEnum: g.emitMessageReadEnum(f, "value."+name, ind) default: @@ -134,7 +175,7 @@ func (g *tableGen) emitMessageReadPayload(f *ir.Field, expr, ind string) { case f.Array != ir.ArrayNone: g.emitMessageReadArrayFrom(f, expr, fmt.Sprintf("value.%s_count", f.Name), ind, "entry", false) case tableScalarKind(f) == tkTable: - g.pf("%sif ( !%s ) { return false; }\n", ind, g.msgLoadCall(f.Type.Name, "r", expr)) + g.pf("%sif ( !%s ) { return false; }\n", ind, g.msgLoadCall(f, f.Type.Name, "r", expr)) case tableScalarKind(f) == tkEnum: g.emitMessageReadEnum(f, expr, ind) default: @@ -210,6 +251,7 @@ func (g *tableGen) emitMessageReadTextFrom(f *ir.Field, value, count, ind, from // discipline over a count rather than a value: a length at or above 2^31 // narrowed FIRST is negative, passes a signed test against the bound // untouched, and lands a negative length in the caller's storage + g.emitRetainReplaced(f, ind+" ") g.pf("%s int32_t kept%s = 0;\n", ind, sfx) g.pf("%s if ( n%s > (uint64_t) %d ) { kept%s = %d; report->clamped++; } else { kept%s = (int32_t) n%s; }\n", ind, sfx, f.Type.Size, sfx, f.Type.Size, sfx, sfx) if f.Type.Kind == ir.TWString { @@ -294,14 +336,26 @@ func (g *tableGen) emitMessageReadArrayFrom(f *ir.Field, base, count, ind, from case tableScalarKind(f) == tkTable: g.pf("%s%s scratch%s;\n", inner, f.Type.Name, sfx) g.pf("%s%sReset( scratch%s );\n", inner, f.Type.Name, sfx) - g.pf("%sif ( !%s ) { return false; }\n", inner, g.msgLoadCall(f.Type.Name, "r", elem)) + g.inStep(idx, func() { + g.inDrop("in_bounds"+sfx, func() { + g.pf("%sif ( !%s ) { return false; }\n", inner, g.msgLoadCall(f, f.Type.Name, "r", elem)) + }) + }) case tableScalarKind(f) == tkEnum: g.pf("%s%s slot_value%s = %s::None;\n", inner, f.Type.Name, sfx, f.Type.Name) g.emitMessageReadEnum(f, "slot_value"+sfx, inner) g.pf("%sif ( in_bounds%s ) { %s[%s] = slot_value%s; }\n", inner, sfx, base, idx, sfx) case tableScalarKind(f) == tkUnion: g.pf("%s%s scratch%s;\n", inner, f.Type.Name, sfx) - g.emitMessageReadUnion(f, elem, inner) + // AN ARRAY OF UNIONS TAKES TWO STEPS AND NOT ONE (docs/SPEC-TABLES.md + // §6.6): the array field's step names the element by its index, and the + // element's own step names the arm by its ordinal. + g.inStep(idx, func() { + wasPath, wasIndex := g.pathExpr, g.elemIndex + g.pathExpr, g.elemIndex = g.step(f), "0" + g.inDrop("in_bounds"+sfx, func() { g.emitMessageReadUnion(f, elem, inner) }) + g.pathExpr, g.elemIndex = wasPath, wasIndex + }) default: // A DISCARDED SURPLUS ELEMENT NEVER ACQUIRES A LIVE DESTINATION. A // fixed-width surplus is stepped over by the arithmetic below and @@ -339,7 +393,11 @@ func (g *tableGen) emitMessageReadEnum(f *ir.Field, dst, ind string) { g.pf("%s TableMessageEntry variant_entry%s;\n", ind, sfx) g.pf("%s if ( variant_ref%s == 0 ) { %s = %s::None; } // the zero reference is the enum's None\n", ind, sfx, dst, e) g.pf("%s else if ( !TableMessageNameEntry( vocabulary, variant_ref%s, variant_entry%s ) ) { report->malformed = true; return false; }\n", ind, sfx, sfx) - g.pf("%s else if ( !TableEnumValue( variant_entry%s.id, %s ) ) { %s = %s::None; report->unknown++; }\n", ind, sfx, dst, dst, e) + // AN UNKNOWN ENUM VARIANT REFERENCE IS AN EXCLUDED CLASS + // (docs/SPEC-TABLES.md §6.6): the FIELD is the reader's, and the reader + // writes its own value under that id, so a retained copy would be a second + // occurrence of an id the reader already wrote. + g.pf("%s else if ( !TableEnumValue( variant_entry%s.id, %s ) ) { %s = %s::None; report->unknown++;%s }\n", ind, sfx, dst, dst, e, g.retainLostMessageInline()) g.pf("%s}\n", ind) } @@ -360,7 +418,14 @@ func (g *tableGen) emitMessageReadUnion(f *ir.Field, dst, ind string) { g.pf("%s else if ( !TableMessageArmEntry( vocabulary, arm_ref%s, %s ) ) { report->malformed = true; return false; }\n", ind, sfx, arm) g.pf("%s else\n%s {\n", ind, ind) g.pf("%s switch ( %s.id )\n%s {\n", ind, arm, ind) - for _, v := range un.Variants { + // THE ARM'S OWN ORDINAL is the element index of a union's step, and not + // "whichever arm is set" (docs/SPEC-TABLES.md §6.6): a caller that switches + // the arm between load and save leaves a step no child body answers, so the + // record is dropped rather than placed in the other arm's body. + wasField := g.unionField + g.unionField = f + defer func() { g.unionField = wasField }() + for ai, v := range un.Variants { mine := ir.TableArmEntry(v) g.noteRef(v.Type) // the arm's id is `mine.Id`, the announcement's own entry, which reads @@ -384,21 +449,35 @@ func (g *tableGen) emitMessageReadUnion(f *ir.Field, dst, ind string) { g.pf("%s if ( !TableMessageSkip( r, vocabulary, index_bits, %s ) ) { report->malformed = true; return false; }\n", ind, arm) g.pf("%s break;\n%s }\n", ind, ind) g.pf("%s %s.type = %sType::%s;\n", ind, dst, un.Name, ir.GoExportName(v.Name)) - switch { - case v.Void(): - case v.Body(): - g.pf("%s if ( !%s ) { return false; }\n", ind, g.msgLoadCall(v.Type, "r", armValue(dst, v))) - default: - g.emitMessageReadArm(v, dst, ind+" ", arm, false) - } + g.inStep(strconv.Itoa(ai), func() { + switch { + case v.Void(): + case v.Body(): + g.pf("%s if ( !%s ) { return false; }\n", ind, g.msgLoadCall(g.unionField, v.Type, "r", armValue(dst, v))) + default: + g.emitMessageReadArm(v, dst, ind+" ", arm, false) + } + }) g.pf("%s break;\n%s }\n", ind, ind) } g.pf("%s default:\n", ind) - g.pf("%s %s.type = %sType::None; report->unknown++;\n", ind, dst, un.Name) + g.pf("%s %s.type = %sType::None; report->unknown++;%s\n", ind, dst, un.Name, g.retainLostMessageInline()) g.pf("%s if ( !TableMessageSkip( r, vocabulary, index_bits, %s ) ) { report->malformed = true; return false; }\n", ind, arm) g.pf("%s break;\n%s }\n%s }\n%s}\n", ind, ind, ind, ind) } +// retainLostMessageInline is one EXCLUDED CLASS counted on a MESSAGE body +// (docs/SPEC-TABLES.md §6.6), emitted into the retain family only and beside +// the unknown the plain read already counts. It is the file form's own +// retainLostInline with this form's report handle: a message body carries the +// report as a parameter where a file's rides on the reader. +func (g *tableGen) retainLostMessageInline() string { + if !g.retain { + return "" + } + return " report->retain_lost++;" +} + func (g *tableGen) emitMessageReadArm(v ir.UnionVariant, base, ind, arm string, widened bool) { af := v.F value, count := armValue(base, v), armCount(base, v) @@ -415,7 +494,9 @@ func (g *tableGen) emitMessageReadArm(v ir.UnionVariant, base, ind, arm string, case tableScalarKind(af) == tkEnum: g.emitMessageReadEnum(af, value, ind) case tableScalarKind(af) == tkTable: - g.pf("%sif ( !%s ) { return false; }\n", ind, g.msgLoadCall(af.Type.Name, "r", value)) + // the step is the UNION FIELD's ordinal and the arm's own, which the + // caller has already put in scope (docs/SPEC-TABLES.md §6.6) + g.pf("%sif ( !%s ) { return false; }\n", ind, g.msgLoadCall(g.unionField, af.Type.Name, "r", value)) default: g.emitMessageReadScalarFromEntry(af, value, ind, arm, false, widened) } @@ -442,7 +523,10 @@ func (g *tableGen) emitMessageReadKeyed(f *ir.Field, ind string) { g.pf("%sif ( !TableMessageNameEntry( vocabulary, key_ref%s, key_entry%s ) ) { report->malformed = true; return false; }\n", inner, sfx, sfx) g.pf("%s%s key%s = %s::None;\n", inner, f.KeyEnum, sfx, f.KeyEnum) g.pf("%sconst bool named%s = TableEnumValue( key_entry%s.id, key%s );\n", inner, sfx, sfx, sfx) - g.pf("%sif ( !named%s ) { report->unknown++; }\n", inner, sfx) + // AN UNKNOWN KEYED-ARRAY SLOT IS AN EXCLUDED CLASS (docs/SPEC-TABLES.md + // §6.6): a slot is not a field, the reader rewrites the array body whole, + // and a slot has nowhere to append to. + g.pf("%sif ( !named%s ) { report->unknown++;%s }\n", inner, sfx, g.retainLostMessageInline()) g.pf("%sconst int32_t slot%s = named%s ? (int32_t) key%s - 1 : -1;\n", inner, sfx, sfx, sfx) g.pf("%sconst bool in_bounds%s = slot%s >= 0 && slot%s < %d;\n", inner, sfx, sfx, sfx, f.ArrayBound) elem := fmt.Sprintf("( in_bounds%s ? %s[slot%s] : scratch%s )", sfx, slots, sfx, sfx) @@ -453,7 +537,14 @@ func (g *tableGen) emitMessageReadKeyed(f *ir.Field, ind string) { case kind == tkTable: g.pf("%s%s scratch%s;\n", inner, f.Type.Name, sfx) g.pf("%s%sReset( scratch%s );\n", inner, f.Type.Name, sfx) - g.pf("%sif ( !%s ) { return false; }\n", inner, g.msgLoadCall(f.Type.Name, "r", elem)) + // THE KEY'S SLOT is the element index of a keyed array's step (§6.6), + // and a slot this build cannot name captures nothing: it was excluded + // whole, and everything under it went with that one retain_lost. + g.inStep("slot"+sfx, func() { + g.inDrop("in_bounds"+sfx, func() { + g.pf("%sif ( !%s ) { return false; }\n", inner, g.msgLoadCall(f, f.Type.Name, "r", elem)) + }) + }) case kind == tkEnum: g.pf("%s%s slot_value%s = %s::None;\n", inner, f.Type.Name, sfx, f.Type.Name) g.emitMessageReadEnum(f, "slot_value"+sfx, inner) @@ -486,6 +577,7 @@ func (g *tableGen) emitMessageReadMap(f *ir.Field, ind string) { g.pf("%s uint64_t count%s = 0;\n", ind, sfx) g.pf("%s if ( !r.get( count%s, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; }\n", ind, sfx) g.pf("%s count%s += (uint64_t) entry.min;\n", ind, sfx) + g.emitRetainReplaced(f, ind+" ") g.pf("%s TableMapFill<%s> fill%s = TableMapFillBegin( nodes, value.%s, (uint32_t) count%s );\n", ind, n, sfx, f.Name, sfx) g.pf("%s if ( !fill%s.ok ) { report->malformed = true; return false; } // the measure and the load disagree\n", ind, sfx) if stringKey { @@ -529,7 +621,11 @@ func (g *tableGen) emitMessageReadMap(f *ir.Field, ind string) { g.pf("%s slot%s = TableMapFillNext( fill%s ); // ASCENDING: the next slot\n", ind, sfx, sfx) g.pf("%s }\n", ind) g.pf("%s if ( slot%s == NULL ) { report->malformed = true; return false; }\n", ind, sfx) - g.pf("%s if ( !%s ) { return false; }\n", ind, g.msgLoadCall(n, "r", "*slot"+sfx)) + // THE KEY'S SLOT is the element index of a map's step, which is the + // entry's index in ascending key order (docs/SPEC-TABLES.md §6.6, §2.8). + g.inStep(fmt.Sprintf("fill%s.map->count - 1", sfx), func() { + g.pf("%s if ( !%s ) { return false; }\n", ind, g.msgLoadCall(f, n, "r", "*slot"+sfx)) + }) g.pf("%s if ( r.offset != read%s.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends\n", ind, sfx) if stringKey { g.pf("%s last_key%s = read%s.key; last_length%s = read%s.length; // the WIRE keys of the entries that LAND\n", ind, sfx, sfx, sfx, sfx) @@ -829,6 +925,7 @@ func (g *tableGen) emitMessageReadList(f *ir.Field, ind string) { if listElementWireKind(f) == tkU8 { g.pf("%s if ( !r.align() ) { report->malformed = true; return false; } // an array of kind 6 aligns before its elements\n", ind) } + g.emitRetainReplaced(f, ind+" ") g.pf("%s TableListFill<%s> fill%s = TableListFillBegin( nodes, value.%s, n%s );\n", ind, g.listTypeArg(f), sfx, f.Name, sfx) g.pf("%s if ( fill%s.refused ) { nodes.refused = true; return false; }\n", ind, sfx) g.pf("%s if ( !fill%s.ok ) { report->malformed = true; return false; } // the measure and the load disagree\n", ind, sfx) @@ -841,7 +938,11 @@ func (g *tableGen) emitMessageReadList(f *ir.Field, ind string) { case f.Type.Pointer: g.emitMessageReadIndex(f, elem, inner) case tableScalarKind(f) == tkTable: - g.pf("%sif ( !%s ) { return false; }\n", inner, g.msgLoadCall(f.Type.Name, "r", elem)) + // THE ELEMENT'S INDEX is the second half of an unbounded array's step + // (docs/SPEC-TABLES.md §6.6, §2.9) + g.inStep(idx, func() { + g.pf("%sif ( !%s ) { return false; }\n", inner, g.msgLoadCall(f, f.Type.Name, "r", elem)) + }) case tableScalarKind(f) == tkEnum: g.emitMessageReadEnum(f, elem, inner) case tableScalarKind(f) == tkUnion: diff --git a/internal/codegen/cpptable/messagevariable.go b/internal/codegen/cpptable/messagevariable.go index dac8ae348..20bd2330b 100644 --- a/internal/codegen/cpptable/messagevariable.go +++ b/internal/codegen/cpptable/messagevariable.go @@ -166,15 +166,32 @@ func (g *tableGen) emitVariableMessageSurface(st *ir.Struct) { g.pf(" if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; }\n") g.pf(" return data + attribution;\n}\n\n") + g.emitVariableMessageLoadSurface(st) +} + +// emitVariableMessageLoadSurface emits the batch's READ half over a pointered +// root: one body into the region, and the batch's own verb over it. It is +// emitted TWICE, once plain and once under RETENTION (docs/SPEC-TABLES.md §3.3, +// §6.6), and the retaining half is the same walk with the caller's stores +// threaded and the unknown arm capturing. +func (g *tableGen) emitVariableMessageLoadSurface(st *ir.Struct) { + n := st.Name // ONE BODY into the region: the directory, the records, the root - g.pf("// %sLoadMessageBodyInto: one body of a batch into the region at `used`. Its\n", n) + g.pf("// %s%s: one body of a batch into the region at `used`. Its\n", n, g.verb("", "LoadMessageBodyInto")) g.pf("// chunk is the node DIRECTORY, then the records in wire order, then the root\n") g.pf("// and the extent its maps take, so every offset a pass needs is known when\n") g.pf("// the pass reaches it. PASS ONE fills the numbering from the framing and\n") g.pf("// places every node; PASS TWO decodes each record's body into the storage it\n") g.pf("// owns; the ROOT's own body decodes last, so every index it carries resolves\n") g.pf("// against a numbering already known whole.\n") - g.pf("inline bool %sLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const %s * & root_out )\n{\n", n, n) + if g.retain { + g.pf("// UNDER RETENTION the body's own retention buffer is reset here and belongs\n") + g.pf("// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because\n") + g.pf("// each body carries its own node directory inside that one region and a\n") + g.pf("// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6).\n") + } + g.pf("inline bool %s( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const %s * & root_out%s )\n{\n", + g.verb(n, "LoadMessageBodyInto"), n, g.retainIntoParam()) g.pf(" // the node table opens the body, or the body has none\n") g.pf(" int64_t count = 0;\n") g.pf(" if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; }\n") @@ -232,7 +249,19 @@ func (g *tableGen) emitVariableMessageSurface(st *ir.Struct) { } g.pf(" used += root_bytes;\n") g.pf(" nodes.good = true;\n") - g.pf(" out->unknown += unknown_records;\n\n") + if g.retain { + g.pf(" // A NODE RECORD whose type id this reader cannot name is one of the SIX\n") + g.pf(" // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means\n") + g.pf(" // renumbering a graph the writer numbers from its own edges.\n") + g.pf(" out->unknown += unknown_records;\n") + g.pf(" out->retain_lost += unknown_records;\n") + g.pf(" // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and\n") + g.pf(" // writes into neither id list, because a retained record carries its\n") + g.pf(" // field's identity in the record itself (§6.6).\n") + g.pf(" TableRetainReset( retain, nodes, region );\n\n") + } else { + g.pf(" out->unknown += unknown_records;\n\n") + } g.pf(" // PASS TWO: each record's body into its own storage, in wire order\n") g.pf(" r.offset = records_start;\n") g.pf(" for ( int64_t k = 0; k < count; k++ )\n {\n") @@ -259,23 +288,40 @@ func (g *tableGen) emitVariableMessageSurface(st *ir.Struct) { g.pf(" if ( directory[k + 1].offset == kTableNodeAbsent )\n {\n") g.pf(" if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; }\n") g.pf(" continue;\n }\n") - g.pf(" if ( !%sNodeMessageBody( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset ) ) { return false; }\n", n) + if g.retain { + g.pf(" if ( !%sNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; }\n", n) + } else { + g.pf(" if ( !%sNodeMessageBody( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset ) ) { return false; }\n", n) + } g.pf(" }\n") g.pf(" if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent\n\n") g.pf(" // and the ROOT's own body last\n") if g.anyExtent { g.pf(" nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's\n") } - g.pf(" return %sLoadMessageBody( r, vocabulary, out, nodes, index_bits, *root );\n}\n\n", n) + if g.retain { + g.pf(" return %sLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) );\n}\n\n", n) + } else { + g.pf(" return %sLoadMessageBody( r, vocabulary, out, nodes, index_bits, *root );\n}\n\n", n) + } // LoadMessages: the batch into ONE region - g.pf("// %sLoadMessages: decode a BATCH into the caller's exact-sized region and\n", n) + g.pf("// %s%s: decode a BATCH into the caller's exact-sized region and\n", n, g.messageBatchVerb()) g.pf("// write each body's root into `roots`. `count` is IN and OUT: the storage the\n") g.pf("// caller has room for, then what it got. M above the capacity is a refusal\n") g.pf("// by name with count holding the wire's M; damage inside body k delivers\n") g.pf("// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows\n") g.pf("// no reference, so there is no depth cap and no visited set. NULL roots\n") g.pf("// beyond count are not bodies.\n") - g.pf("inline bool %sLoadMessages( const %s ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report )\n{\n", n, n) + if g.retain { + g.pf("// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6):\n") + g.pf("// `retains` is an array parallel to `roots`, each entry reset by this call\n") + g.pf("// and each holding the records of the body it belongs to, which is what keeps\n") + g.pf("// a record's first step an index into that body's own node directory. A\n") + g.pf("// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own\n") + g.pf("// pair unchanged: retention writing FORM 2 refuses by name (§3.3).\n") + } + g.pf("inline bool %s( const %s ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes,%s TableReport * report )\n{\n", + g.messageBatchVerbOf(n), n, g.retainBatchParam()) g.pf(" TableReport ignored;\n") g.pf(" TableReport * out = report != NULL ? report : &ignored;\n") g.pf(" if ( roots == NULL || count == NULL ) { out->malformed = true; return false; }\n") @@ -290,13 +336,52 @@ func (g *tableGen) emitVariableMessageSurface(st *ir.Struct) { g.pf(" int64_t used = 0;\n") g.pf(" for ( int64_t b = 0; b < bodies; b++ )\n {\n") g.pf(" roots[b] = NULL;\n") - g.pf(" if ( !%sLoadMessageBodyInto( br.r, vocabulary, out, region, region_bytes, used, roots[b] ) ) { *count = b; return false; }\n", n) + if g.retain { + g.pf(" if ( !%s( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; }\n", + g.verb(n, "LoadMessageBodyInto")) + } else { + g.pf(" if ( !%sLoadMessageBodyInto( br.r, vocabulary, out, region, region_bytes, used, roots[b] ) ) { *count = b; return false; }\n", n) + } g.pf(" br.remaining--;\n }\n") g.pf(" *count = bodies;\n") g.pf(" return TableMessageBatchClose( br );\n}\n\n") } +// messageBatchVerb is the batch's own READ verb under the family being +// emitted. The page spells no name for the form-2 retaining read: §6.6 and §3.3 +// name it "the message form's LoadRetain" and §11's claimed set carries +// `SaveRetainMessages`, the write refusal, and no read spelling beside it. The +// reduce-cases name is the one this claims, `LoadRetainMessages` beside +// `LoadMessages`, and it is claimed in `tableGeneratedVerbs` and §11 on that +// list's own rule, because it is EMITTED. +func (g *tableGen) messageBatchVerb() string { + if g.retain { + return "LoadRetainMessages" + } + return "LoadMessages" +} + +func (g *tableGen) messageBatchVerbOf(name string) string { return name + g.messageBatchVerb() } + +// retainIntoParam is the one argument a retaining body-into takes: THIS BODY's +// own retention buffer (docs/SPEC-TABLES.md §3.3, §6.6). +func (g *tableGen) retainIntoParam() string { + if g.retain { + return ", TableRetain * retain" + } + return "" +} + +// retainBatchParam is the array of them the batch takes, parallel to the +// caller's array of roots. +func (g *tableGen) retainBatchParam() string { + if g.retain { + return " TableRetain * retains," + } + return "" +} + // emitRootNodeMessageDispatch emits the three answers a MESSAGE load needs // about a wire type id, over the members this root's numbering can name // (docs/SPEC-TABLES.md §3.1, §3.3): the storage a record commands, the extent @@ -306,6 +391,13 @@ func (g *tableGen) emitRootNodeMessageDispatch(st *ir.Struct) { reachable := ir.PointerReachable(st) blobs := reachableBlobs(st) + if g.retain { + // THE STORAGE AND THE EXTENT ARE THE PLAIN FAMILY'S, unchanged: they + // read the framing and no body, so retention has nothing to say about + // either and the retain pass emits the BODY dispatch alone. + g.emitRootNodeMessageBodyDispatch(st, reachable) + return + } g.pf("// %sNodeMessageStorage: the region bytes one record commands on the message\n", n) g.pf("// wire, or -1 for a type id this build cannot name. A table's is its own\n") g.pf("// storage plus the extent its maps take; a byte buffer's is its header and\n") @@ -351,9 +443,23 @@ func (g *tableGen) emitRootNodeMessageDispatch(st *ir.Struct) { } g.pf(" return TableMessageSkipBody( r, vocabulary, index_bits );\n}\n\n") - g.pf("// %sNodeMessageBody: PASS TWO's half, which decodes one record's body into\n", n) + g.emitRootNodeMessageBodyDispatch(st, reachable) +} + +// emitRootNodeMessageBodyDispatch is PASS TWO's half of the dispatch, emitted +// once per family: the plain one and, under retention, the same switch with the +// caller's store and this node's own path root threaded through it. +func (g *tableGen) emitRootNodeMessageBodyDispatch(st *ir.Struct, reachable []*ir.Struct) { + n := st.Name + g.pf("// %s%s: PASS TWO's half, which decodes one record's body into\n", n, g.verb("", "NodeMessageBody")) g.pf("// storage it already owns, its map entries carved from its own extent.\n") - g.pf("inline bool %sNodeMessageBody( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at )\n{\n", n) + if g.retain { + g.pf("// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the\n") + g.pf("// index is this body's own directory position, which the batch's load fills\n") + g.pf("// from the framing and nothing afterwards renumbers.\n") + } + g.pf("inline bool %s( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at%s )\n{\n", + g.verb(n, "NodeMessageBody"), g.retainNodeBodyParams()) // A LIST CARVES FROM THE NODE'S EXTENT EXACTLY AS A MAP DOES // (docs/SPEC-TABLES.md §2.8, §2.9), so the cursor rides wherever the unit // has an extent at all and not only where it has a map. @@ -383,10 +489,22 @@ func (g *tableGen) emitRootNodeMessageDispatch(st *ir.Struct) { if !anyVar { g.pf(" (void) nodes; (void) index_bits; // every node this root can name is a FIXED table\n") } + if g.retain && len(reachable) == 0 { + // a root whose numbering can name no TABLE record: a blob's bytes carry + // no body, so there is nothing under this node for a path to reach + g.pf(" (void) retain; (void) node;\n") + } g.pf(" bool ok = false;\n") g.pf(" switch ( type_id )\n {\n") for _, t := range reachable { - g.pf(" case 0x%016xull: ok = %s; break; // %s\n", ir.TableWireId(t.Name), g.msgLoadCall(t.Name, "r", fmt.Sprintf("*(%s *) at", t.Name)), t.Name) + call := g.msgLoadCall(nil, t.Name, "r", fmt.Sprintf("*(%s *) at", t.Name)) + if g.retain { + call = fmt.Sprintf("%sLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(%s *) at, retain, TableRetainPathRoot( (const void *) at, node ) )", t.Name, t.Name) + if !g.isVar(t.Name) { + call = fmt.Sprintf("%sLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(%s *) at, retain, TableRetainPathRoot( (const void *) at, node ) )", t.Name, t.Name) + } + } + g.pf(" case 0x%016xull: ok = %s; break; // %s\n", ir.TableWireId(t.Name), call, t.Name) } g.pf(" // a record this dispatch cannot name never reaches here: pass one left it absent\n") g.pf(" default: report->malformed = true; break;\n }\n") diff --git a/internal/codegen/cpptable/retain.go b/internal/codegen/cpptable/retain.go index 583f79c73..1fa7870f2 100644 --- a/internal/codegen/cpptable/retain.go +++ b/internal/codegen/cpptable/retain.go @@ -1258,6 +1258,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} ` // ---- the emitted surface (docs/SPEC-TABLES.md §6.6) ---- @@ -1292,11 +1724,13 @@ func (g *tableGen) emitRetainDeclarations(members []*ir.Struct) { g.pf("template inline bool %sSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const %s & value, TableRetain * retain, const TableRetainPath & path );\n", st.Name, st.Name) g.pf("template inline bool %sSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const %s & value, TableRetain * retain, const TableRetainPath & path );\n", st.Name, st.Name) g.pf("inline bool %sLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, %s & value, TableRetain * retain, const TableRetainPath & path );\n", st.Name, st.Name) + g.pf("inline bool %sLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, %s & value, TableRetain * retain, const TableRetainPath & path );\n", st.Name, st.Name) continue } g.pf("inline int64_t %sMeasureBodyRetain( TableRetainIds & ids, const %s & value, TableRetain * retain, const TableRetainPath & path );\n", st.Name, st.Name) g.pf("%s bool %sSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const %s & value, TableRetain * retain, const TableRetainPath & path );\n", tableInlineMacro(g.unit.Package), st.Name, st.Name) g.pf("%s bool %sLoadBodyRetain( TableReader & r, %s & value, TableRetain * retain, const TableRetainPath & path );\n", tableInlineMacro(g.unit.Package), st.Name, st.Name) + g.pf("inline bool %sLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, %s & value, TableRetain * retain, const TableRetainPath & path );\n", st.Name, st.Name) } g.pf("\n") } @@ -1310,6 +1744,11 @@ func (g *tableGen) emitRetainBodies(members []*ir.Struct) { g.emitTableMeasure(st) g.emitTableWrite(st) g.emitTableRead(st) + // AND THE MESSAGE FORM'S OWN BODY beside it (docs/SPEC-TABLES.md §3.3): + // a form-2 read retains where a form-2 write refuses by name, so the + // LOAD half of that family is emitted here and the measure and the save + // are not. + g.emitMessageLoadBody(st) } g.retain = false g.setOwner(nil) @@ -1365,6 +1804,12 @@ func (g *tableGen) emitRetainRoot(st *ir.Struct) { g.retain = true g.emitRootNodeBody(st, reachable, blobs) g.emitVariableLoad(st) + // THE MESSAGE FORM'S LoadRetain (docs/SPEC-TABLES.md §3.3): the batch's + // read half over this root, with the caller's stores threaded. There is no + // measure and no save beside it, because retention writing form 2 refuses + // by name and the file form's own pair is what a caller saves through. + g.emitRootNodeMessageDispatch(st) + g.emitVariableMessageLoadSurface(st) g.retain = false g.pf("// %sMeasureRetain and %sSaveRetain: the pair, with the retained tail in\n", n, n) diff --git a/internal/tablewire/messagedecode.go b/internal/tablewire/messagedecode.go index d00371b30..9a6056de8 100644 --- a/internal/tablewire/messagedecode.go +++ b/internal/tablewire/messagedecode.go @@ -65,6 +65,14 @@ func MessageCount(data []byte, v *Vocabulary) (int, error) { // is read. The storage for body `k` holds whatever the decode had put in // it, and the COUNT is what says it is not a body. func DecodeMessages(m *tabletext.Model, insts []*tabletext.Instance, data []byte, v *Vocabulary, report *tabletext.Report) (int, bool, error) { + return decodeMessagesWith(m, insts, data, v, report, nil) +} + +// decodeMessagesWith is the batch read under an optional RETENTION STATE A +// BODY (docs/SPEC-TABLES.md §6.6, retainmessage.go). `states` is nil for the +// plain read and holds one entry per body for the retaining one, so the plain +// path is the path it always was and retains nothing. +func decodeMessagesWith(m *tabletext.Model, insts []*tabletext.Instance, data []byte, v *Vocabulary, report *tabletext.Report, states []*retainState) (int, bool, error) { // THE FORM BYTE IS READ FIRST, before the count and before any body, so a // wire that is both a form this reader does not carry and damaged is a // refusal and never damage (§3). @@ -95,6 +103,9 @@ func DecodeMessages(m *tabletext.Model, insts []*tabletext.Instance, data []byte } for i := range count { d := &bitDecoder{m: m, v: v, report: report, r: r, refBits: v.RefBits(), indexBits: ir.TableMessageBitsRequired(0, 1)} + if i < len(states) { + d.rt = states[i] + } if !d.root(insts[i]) { return i, false, nil } @@ -136,6 +147,26 @@ type bitDecoder struct { // many entries carry it (§2.8, §4). It is consumed by the entry body it // was set for, so a nested body never sees it. mapKey *bool + // rt, when set, is THIS BODY's retention state (§6.6): the caller's two + // stores and the ids this build can spell. A batch holds one a body, + // because a body carries its own node directory inside the batch's one + // region and a record's first step is an index into it (§3.3). + rt *retainState +} + +// dropped runs one element's read with RETENTION OFF (docs/SPEC-TABLES.md +// §6.6). A bit stream has to be walked past, so an element the reader is +// dropping, an unknown keyed slot's, a keyed key past this build's slots, an +// array element past this build's bound, is DECODED into a sink where a file's +// reader steps over it by its length. Nothing in that body is a field of this +// region: the slot or the element is what was excluded, and everything under it +// went with it under that one `retain_lost`. +func (d *bitDecoder) dropped(read func() bool) bool { + was := d.rt + d.rt = nil + ok := read() + d.rt = was + return ok } // BitSpot is one number's place in a batch's bit stream: a reference at @@ -450,10 +481,17 @@ func (d *bitDecoder) body(inst *tabletext.Instance) bool { i, known := index[entry.Id] if !known { d.report.Unknown++ + // THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits + // (retainmessage.go): the plain read's verdict on this entry is the + // skip's, whether or not this build retains. + at := d.r.off if !d.skip(entry) { d.report.Malformed = true return false } + if d.rt != nil { + d.rt.captureMessage(d, inst, entry, at, d.r.off) + } continue } fv := &inst.Fields[i] @@ -489,6 +527,11 @@ func (d *bitDecoder) body(inst *tabletext.Instance) bool { } continue } + if discardsWhole(fv.Def) { + // THE FIELD IS BEING READ AGAIN (docs/SPEC-TABLES.md §6.6): every + // record under it goes before the winning occurrence is read. + d.rt.forgetField(fv) + } if !d.field(fv, entry) { return false } @@ -702,6 +745,13 @@ func (d *bitDecoder) array(fv *tabletext.Field, entry ir.TableVocabularyEntry) b // exists to find the bit the array ENDS at, and where the element's width // is announced that bit is a multiplication rather than a loop, which is // also what keeps a zero-width element under a wide count bounded. + // A SECOND OCCURRENCE OF A COLLECTION REPLACES IT WHOLE, and it takes its + // discard HERE rather than at the field: the commitment is past the count + // and the align, and a header that turned the occurrence inert would have + // left the field standing with its records (docs/SPEC-TABLES.md §6.6). + for i := range fv.Elems { + d.rt.forgetCellRecords(&fv.Elems[i]) + } walk := n if run >= 0 && walk > kept { walk = kept @@ -709,13 +759,21 @@ func (d *bitDecoder) array(fv *tabletext.Field, entry ir.TableVocabularyEntry) b for i := uint64(0); i < walk; i++ { var sink tabletext.Cell cell := &sink - if f.Array == ir.ArrayList { + mine := true + switch { + case f.Array == ir.ArrayList: fv.Elems = append(fv.Elems, d.m.ElementZero(f)) cell = &fv.Elems[i] - } else if i < kept { + case i < kept: cell = &fv.Elems[i] + default: + mine = false // an element past this build's bound: read into a sink and dropped } - if !d.element(cell, f, shape) { + element := func() bool { return d.element(cell, f, shape) } + if !mine { + element = func() bool { return d.dropped(func() bool { return d.element(cell, f, shape) }) } + } + if !element() { return false } } @@ -740,6 +798,7 @@ func (d *bitDecoder) element(cell *tabletext.Cell, f *ir.Field, shape ir.TableMe } switch int(shape.Elem) { case ir.TableKindTable: + d.rt.forgetCellRecords(cell) // the element's earlier body goes with it (§6.6) cell.Tab = d.m.New(tabletext.StructOf(f)) return d.body(cell.Tab) case ir.TableKindPointer: @@ -785,15 +844,33 @@ func (d *bitDecoder) keyed(fv *tabletext.Field, entry ir.TableVocabularyEntry) b value := enumValueForId(f.KeyEnumRef, key.Id) var sink tabletext.Cell cell := &sink + kept := true switch { case value < 0: d.report.Unknown++ + if d.rt != nil { + // AN UNKNOWN KEYED-ARRAY SLOT IS AN EXCLUDED CLASS + // (docs/SPEC-TABLES.md §6.6): a slot is not a field, the reader + // rewrites the array body whole, and a slot has nowhere to + // append to. + d.rt.lost(d.report) + } + kept = false case value > int64(tabletext.KeyedSlotCount(f)): d.report.Clamped++ + kept = false default: + // A KEYED-ARRAY SLOT WRITTEN AGAIN is one of §6.6's four + // occurrences, and only the slots this occurrence carries are + // replaced: the rest stand, and so do their records. The discard + // is the element's own, below. cell = &fv.Elems[value-1] } - if !d.element(cell, f, shape) { + element := func() bool { return d.element(cell, f, shape) } + if !kept { + element = func() bool { return d.dropped(func() bool { return d.element(cell, f, shape) }) } + } + if !element() { return false } } @@ -819,6 +896,12 @@ func (d *bitDecoder) mapField(fv *tabletext.Field, entry ir.TableVocabularyEntry d.report.Malformed = true return false } + // A SECOND OCCURRENCE OF A MAP REPLACES IT WHOLE (docs/SPEC-TABLES.md + // §6.6): every record under an entry the earlier occurrence held goes with + // the entry, and the discard moves neither counter. + for i := range fv.Entries { + d.rt.forgetCellRecords(&fv.Entries[i]) + } fv.Entries = nil var last *tabletext.Instance // A KEY KIND THIS DECLARATION WIDENS IS NOT A DISAGREEMENT (§2.8, §4): @@ -835,7 +918,10 @@ func (d *bitDecoder) mapField(fv *tabletext.Field, entry ir.TableVocabularyEntry } key := tabletext.MapKeyOf(f, decoded) if !tabletext.MapKeyFits(f, key) { - // KEYS NEVER CLAMP: the entry is dropped whole, one count per entry + // KEYS NEVER CLAMP: the entry is dropped whole, one count per entry. + // A record the decode captured under it is one the save's walk will + // not reach, and it counts its `retain_lost` THERE, exactly as the + // file form's own dropped entry does (§6.6). d.report.Clamped++ continue } @@ -849,6 +935,9 @@ func (d *bitDecoder) mapField(fv *tabletext.Field, entry ir.TableVocabularyEntry } if order == 0 { // EQUAL: a DUPLICATE. Last wins WHOLE, and the count excludes it. + // A MAP'S DUPLICATE KEY is one of §6.6's four occurrences: the + // records the slot's earlier entry held go with it. + d.rt.forgetCellRecords(&fv.Entries[len(fv.Entries)-1]) fv.Entries[len(fv.Entries)-1] = tabletext.Cell{Tab: decoded} d.report.Duplicate++ last = decoded @@ -870,6 +959,10 @@ func (d *bitDecoder) mapField(fv *tabletext.Field, entry ir.TableVocabularyEntry // arm's payload is skipped by the ARM's own announced entry. func (d *bitDecoder) unionCell(cell *tabletext.Cell, f *ir.Field) bool { un := tabletext.UnionOf(f) + // A UNION WHOSE ARM IS WRITTEN AGAIN, THE SAME ARM OR ANOTHER, is one of + // §6.6's four occurrences: the earlier arm body's records go with it, and + // the discard moves neither counter. + d.rt.forgetArm(cell) ref, ok := d.reference() if !ok { d.report.Malformed = true @@ -896,6 +989,12 @@ func (d *bitDecoder) unionCell(cell *tabletext.Cell, f *ir.Field) bool { if tag == 0 { cell.U = 0 d.report.Unknown++ + if d.rt != nil { + // AN UNKNOWN UNION ARM ID IS AN EXCLUDED CLASS + // (docs/SPEC-TABLES.md §6.6): the same as an unknown enum variant, + // one level in, because the union FIELD is the reader's. + d.rt.lost(d.report) + } return d.skip(entry) } arm := un.Variants[tag-1] @@ -916,6 +1015,7 @@ func (d *bitDecoder) unionCell(cell *tabletext.Cell, f *ir.Field) bool { return true case arm.Body(): cell.Tab = d.m.New(arm.Ref) + d.rt.builtArm(cell, cell.Tab) return d.body(cell.Tab) } fv := d.m.NewArm(arm) @@ -968,6 +1068,13 @@ func (d *bitDecoder) enumCell(cell *tabletext.Cell, f *ir.Field) bool { if v < 0 { cell.U = 0 d.report.Unknown++ + if d.rt != nil { + // AN UNKNOWN ENUM VARIANT REFERENCE IS AN EXCLUDED CLASS + // (docs/SPEC-TABLES.md §6.6): the FIELD is the reader's, and the + // reader writes its own value under that id, so a retained copy + // would be a second occurrence of an id the reader already wrote. + d.rt.lost(d.report) + } return true } cell.U = uint64(v) diff --git a/internal/tablewire/retain.go b/internal/tablewire/retain.go index fd5d54c16..abbfc0d93 100644 --- a/internal/tablewire/retain.go +++ b/internal/tablewire/retain.go @@ -160,6 +160,15 @@ func (rt *retainState) forget(inst *tabletext.Instance) { } } +// forgetCellRecords is forgetCell with the nil-state check the exported +// entry points carry, for a caller that discards one cell rather than a body. +func (rt *retainState) forgetCellRecords(cell *tabletext.Cell) { + if rt == nil { + return + } + rt.forgetCell(cell) +} + func (rt *retainState) forgetCell(cell *tabletext.Cell) { if cell.Tab != nil { rt.forget(cell.Tab) @@ -561,11 +570,20 @@ func (rt *retainState) capture(inst *tabletext.Instance, id uint64, kind uint8, rt.lost(report) return } - // THE EXPANSION IS BOUNDED, AND THIS ENGINE STATES THE CONSTANT: a record - // costs the id it rides under, its kind byte and the resolved payload, - // which is the wire's bytes plus seven for each reference widened to eight - // and this engine's own fixed-width inner lengths (§6.6). - cost := 9 + len(rs.out) + rt.keep(inst, id, kind, rs.out, report) +} + +// keep is the CAPACITY and the counter, which every capture answers the same +// way whatever form it read from: the file's resolving walk above and the +// message form's transcoding one (retainmessage.go) both land here, so the +// accounting is stated once. +// +// THE EXPANSION IS BOUNDED, AND THIS ENGINE STATES THE CONSTANT: a record costs +// the id it rides under, its kind byte and the resolved payload, which is the +// wire's bytes plus seven for each reference widened to eight and this engine's +// own fixed-width inner lengths (§6.6). +func (rt *retainState) keep(inst *tabletext.Instance, id uint64, kind uint8, payload []byte, report *tabletext.Report) { + cost := 9 + len(payload) if !rt.store.fits(cost) { // REFUSAL IS PER RECORD AND NEVER PARTIAL: the record is not written // at all, the read continues, and the buffer never holds a truncated @@ -574,7 +592,7 @@ func (rt *retainState) capture(inst *tabletext.Instance, id uint64, kind uint8, rt.lost(report) return } - rt.store.records[inst] = append(rt.store.records[inst], retainedField{id: id, kind: kind, payload: rs.out, cost: cost}) + rt.store.records[inst] = append(rt.store.records[inst], retainedField{id: id, kind: kind, payload: payload, cost: cost}) report.Retained++ } diff --git a/internal/tablewire/retain_test.go b/internal/tablewire/retain_test.go index 0aaecdaae..fc455ad80 100644 --- a/internal/tablewire/retain_test.go +++ b/internal/tablewire/retain_test.go @@ -685,3 +685,92 @@ func TestRetainDroppedRecordSpendsNoId(t *testing.T) { } } } + +// THE MESSAGE FORM'S LoadRetain (docs/SPEC-TABLES.md §3.3, §6.6). `LoadRetain` +// reads a form-2 body as it reads a file's, the resolving walk replacing every +// reference with the id it names AGAINST THE CONNECTION'S VOCABULARY instead +// of a trailer, and `SaveRetain` writing form 2 refuses by name. So the round +// trip is form 2 in and form 1 out, and this row is the same pinned batch the +// reference's own gate reads, saved to the same bytes. +// +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3): each body +// carries its own node directory inside that one region, so a record's first +// step is an index into the directory of the body it came from. +func TestRetainMessageForm(t *testing.T) { + m := retainModel(t, "RT1.schema") + sender := retainModel(t, "RT2.schema") + + v := &tablewire.Vocabulary{} + var announced tabletext.Report + if err := v.AnnounceRead(ir.TableAnnouncement(sender.Unit), &announced); err != nil { + t.Fatalf("RT2's own announcement was refused: %v", err) + } + + batch := retainVector(t, "retain_message") + insts := []*tabletext.Instance{m.New(m.Lookup("Node")), m.New(m.Lookup("Node"))} + retains := []*tablewire.Retain{ + {Capacity: 8192, IdCapacity: 1024}, + {Capacity: 8192, IdCapacity: 1024}, + } + var report tabletext.Report + read, ok, err := tablewire.DecodeRetainMessages(m, insts, batch, v, retains, &report) + if err != nil { + t.Fatal(err) + } + if !ok || report.Malformed { + t.Fatalf("the load reported damage on a sound batch: %+v", report) + } + if read != 2 { + t.Fatalf("the batch carries 2 bodies and the load read %d", read) + } + // THE FIRST BODY carries eight retained fields and six unknowns of the + // excluded classes: the enum variant `tier` names, the union arm `pick` + // names, the keyed slot RT2's third variant writes, a field of kind 17, an + // array whose element kind is 17, and a table whose payload meets a 17 + // three bodies down. THE SECOND carries two retained fields and nothing + // excluded. + // + // `unknown` IS SEVENTEEN AND NOT SIXTEEN, and the seventeenth is the plain + // read's own: a bit stream has to be walked past, so the DISCARDED SLOT's + // element body is decoded into a sink where a file's reader steps over it + // by its length, and the field inside it that this build cannot name counts + // there. Retention keeps nothing under it, the slot is what was excluded + // and everything below it went with that one `retain_lost`, so the two + // retention counters are the numbers the file form gives. + if report.Retained != 10 || report.RetainLost != 6 || report.Unknown != 17 { + t.Fatalf("retained=%d retain_lost=%d unknown=%d, want 10 / 6 / 17", + report.Retained, report.RetainLost, report.Unknown) + } + if report.KindMismatch != 0 || report.Clamped != 0 || report.Widened != 0 { + t.Fatalf("retention moved a read counter: %+v", report) + } + // AND THE PLAIN READ OF THE SAME BATCH SAYS THE SAME SEVENTEEN, which is + // what makes that line the READ's and not retention's: the three names are + // ADDITIVE and retention moves no existing counter (§6.6). + var plain tabletext.Report + if _, _, err := tablewire.DecodeMessages(m, []*tabletext.Instance{ + m.New(m.Lookup("Node")), m.New(m.Lookup("Node")), + }, batch, v, &plain); err != nil { + t.Fatal(err) + } + if plain.Unknown != report.Unknown || plain.Retained != 0 || plain.RetainLost != 0 { + t.Fatalf("the plain read says %+v beside the retaining read's %+v", plain, report) + } + + // AND THE SAVE IS THE FILE FORM'S, byte for byte the reference's own pin + for i, inst := range insts { + var save tabletext.Report + out, err := tablewire.EncodeRetain(m, inst, retains[i], &save) + if err != nil { + t.Fatal(err) + } + if save.RetainLost != 0 { + t.Fatalf("body %d: the save lost %d records it had room for", i, save.RetainLost) + } + want := retainVector(t, []string{"retain_message_save_0", "retain_message_save_1"}[i]) + if !bytes.Equal(out, want) { + t.Fatalf("body %d: the save is %d bytes and the reference's pin is %d, and they differ", + i, len(out), len(want)) + } + } +} diff --git a/internal/tablewire/retainmessage.go b/internal/tablewire/retainmessage.go new file mode 100644 index 000000000..a541f8806 --- /dev/null +++ b/internal/tablewire/retainmessage.go @@ -0,0 +1,496 @@ +// RETAIN-UNKNOWN ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3, §6.6). +// +// `LoadRetain` reads a form-`2` body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and `SaveRetain` writing form `2` refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the round trip a +// caller has is form 2 in and form 1 out: the file carries its own table and +// takes §6.6 unchanged. +// +// THE RECORD IS THE FILE FORM'S OWN, to the byte. A retained record must carry +// the field's identity and bytes WITH EVERY REFERENCE RESOLVED, so that +// re-emitting it into any id table is correct, and the table it is re-emitted +// into is a FILE's. So the capture here is a TRANSCODE as well as a resolve: a +// bitpacked payload is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record the +// file's own capture makes and the same emitter writes back. Nothing below +// copies the emit side, and nothing below is a second record layout. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's own verdict on an unknown entry is the skip's. A variant reference +// that names an entry carrying a payload is damage whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture then re-reads the bits the skip +// delimited, and a capture that lands anywhere but the skip's own end drops the +// record: two engines that disagree about a payload's width would otherwise +// disagree in silence. It is two flat passes over one record's bits and never a +// walk that doubles at every level, which is what the cost rule forbids (§6.6). +package tablewire + +import ( + "encoding/binary" + "math" + "math/big" + + "github.com/mas-bandwidth/schema/v2/internal/tabletext" + "github.com/mas-bandwidth/schema/v2/ir" +) + +// DecodeRetainMessages is [DecodeMessages] with retention on +// (docs/SPEC-TABLES.md §3.3, §6.6). It reads exactly what [DecodeMessages] +// reads and reports exactly what it reports, and beside that it keeps the +// fields this build cannot name, in the caller's own buffers. +// +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY. The batch is one +// region and each body carries its OWN node directory inside it (§3.3), so a +// record's first step is an index into the directory of the body it came from, +// and the buffer that holds it is that body's own: `retains[k]` belongs to +// `insts[k]`, is reset by this call, and is what `EncodeRetain` from that +// instance reads. A caller with fewer buffers than the batch has bodies is +// refused before a body is read, on the batch bound's own terms. +func DecodeRetainMessages(m *tabletext.Model, insts []*tabletext.Instance, data []byte, v *Vocabulary, retains []*Retain, report *tabletext.Report) (int, bool, error) { + if retains == nil { + return DecodeMessages(m, insts, data, v, report) + } + if len(retains) < len(insts) { + insts = insts[:len(retains)] + } + states := make([]*retainState, len(insts)) + for i := range states { + if retains[i] != nil { + states[i] = newRetainState(m, retains[i]) + } + } + return decodeMessagesWith(m, insts, data, v, report, states) +} + +// captureMessage is the whole of the load side for one unknown entry: the +// outer-kind exclusion, the transcoding walk, the capacity, and the counter +// each answers. `at` is where the payload began and `end` where the skip left +// the reader, and the walk must land on `end` exactly. +func (rt *retainState) captureMessage(d *bitDecoder, inst *tabletext.Instance, entry ir.TableVocabularyEntry, at, end int) { + if entry.Kind == ir.TableKindPointer { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + rt.lost(d.report) + return + } + rs := &resolver{} + was := d.r.off + d.r.off = at + rs.msgPayload(d, entry, 0) + landed := d.r.off + d.r.off = was + if rs.bad || landed != end { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // `malformed` DOES NOT MOVE (§6.6). + rt.lost(d.report) + return + } + rt.keep(inst, entry.Id, entry.Kind, rs.out, d.report) +} + +// msgRef resolves one reference against the CONNECTION'S VOCABULARY and answers +// the sixty-four-bit id it names. It is the file walk's own verdict read +// against an announcement instead of a trailer (§6.6): a reference of zero, one +// above the entry count, and one naming a RESERVED id, which would be +// re-emitted into a nested body where it is malformed, all drop the record. +func (rs *resolver) msgRef(d *bitDecoder) uint64 { + ref, ok := d.reference() + if !ok { + rs.bad = true + return 0 + } + return rs.msgId(d, ref) +} + +func (rs *resolver) msgId(d *bitDecoder, ref uint64) uint64 { + entry, named := d.entry(ref) + if !named || reserved(entry.Id) || entry.Id == 0 { + rs.bad = true + return 0 + } + return entry.Id +} + +// msgPayload is the resolving walk over ONE announced payload, a field's, an +// array element's, a union arm's, written out in the FILE form's own spelling. +// +// WHICH KINDS THE WALK TOUCHES is §6.6's list unchanged: kind 13, a body's +// fields; kind 15, an arm id and then the arm's payload under this same rule; +// kind 30, a variant id; kind 14 at its element kind; and kind 16 at EVERY +// element kind, because a keyed body carries a KEY REFERENCE per slot whatever +// the elements are. What the file form COPIES VERBATIM is transcoded here +// instead, because a bitpacked value is not the file's bytes. +// +// KIND 17 IS WHAT THE WALK IS LOOKING FOR AS MUCH AS A REFERENCE IS: meeting +// one anywhere in the payload drops the whole record, which is what keeps a +// retained record from ever carrying a node index. +func (rs *resolver) msgPayload(d *bitDecoder, entry ir.TableVocabularyEntry, depth int) { + if rs.bad { + return + } + switch int(entry.Kind) { + case ir.TableKindPointer: + // THE NODE-INDEX CLASS, met inside a payload rather than at the outer + // kind: the record is dropped whole (§6.6) + rs.bad = true + case 0: + // A KIND-0 ENTRY IS A NAME (§3.3) and names no payload the file form + // has a kind for, so a body that used one as a field is a shape this + // walk cannot frame: the plain read skips it and this drops it. + rs.bad = true + case ir.TableKindNoPayload: + // kind 32 rides no bits here and an empty framed payload on the file + rs.out = appendLeb(rs.out, 0) + case ir.TableKindEnum: + ref, ok := d.reference() + if !ok { + rs.bad = true + return + } + if ref == 0 { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, not + // a reference, and it rides back as one + rs.u64(0) + return + } + rs.u64(rs.msgId(d, ref)) + case ir.TableKindUnion: + ref, ok := d.reference() + if !ok { + rs.bad = true + return + } + if ref == 0 { + rs.u64(0) + return + } + arm, named := d.entry(ref) + if !named || reserved(arm.Id) || arm.Kind == 0 { + rs.bad = true + return + } + rs.u64(arm.Id) + rs.u8(arm.Kind) + rs.msgFramed(d, arm, depth) + case ir.TableKindTable, ir.TableKindArray, ir.TableKindKeyed: + rs.msgFramed(d, entry, depth) + case ir.TableKindString, ir.TableKindWstring, ir.TableKindEscape: + // an opaque payload rides behind a CANONICAL LEB128 LENGTH on the file + b := rs.msgOpaque(d, entry) + if rs.bad { + return + } + rs.out = appendLeb(rs.out, uint64(len(b))) + rs.out = append(rs.out, b...) + default: + rs.msgScalar(d, entry) + } +} + +// msgOpaque reads one text or blob payload and answers its FILE bytes. Each of +// the three has its own framing here and none of them on the file, where the +// payload's own length frames it: a `string(N)` or a `bytes(N)` is a length at +// its own width, the ALIGN that buys a memcpy, then the bytes; a wide string is +// a length and then SIXTEEN bits a code unit, which is two file bytes each; and +// the ESCAPE aligns, reads a thirty-two bit L and takes L bytes opaque (§3.3). +func (rs *resolver) msgOpaque(d *bitDecoder, entry ir.TableVocabularyEntry) []byte { + n := 0 + switch int(entry.Kind) { + case ir.TableKindString, ir.TableKindWstring: + raw, ok := d.r.get(ir.TableMessageBitsRequired(0, entry.Shape.Max)) + if !ok { + rs.bad = true + return nil + } + n = int(raw) + if entry.Kind == ir.TableKindWstring { + n *= 2 + } else if !d.r.align() { + rs.bad = true + return nil + } + default: + if !d.r.align() { + rs.bad = true + return nil + } + raw, ok := d.r.get(32) + if !ok { + rs.bad = true + return nil + } + n = int(raw) + } + b, ok := d.r.bytes(n) + if !ok { + rs.bad = true + return nil + } + return b +} + +// msgFramed writes one CONTENT behind a length. The lengths inside a record are +// this engine's own spelling, so each is written after the content it frames +// and the capture costs one pass (§6.6). +func (rs *resolver) msgFramed(d *bitDecoder, entry ir.TableVocabularyEntry, depth int) { + at := rs.mark() + rs.msgContent(d, entry, depth+1) + rs.patch(at) +} + +// msgContent resolves a payload the file form frames by a length: a kind 13, +// 14 or 16 field's own body, a union arm's payload, and an enum-keyed slot's +// element. +func (rs *resolver) msgContent(d *bitDecoder, entry ir.TableVocabularyEntry, depth int) { + if rs.bad { + return + } + if depth > retainDepthCap { + // THE WALK'S NESTING CAP: a record past it is dropped on the same rule + // as any other shape the walk cannot take (§6.6) + rs.bad = true + return + } + switch int(entry.Kind) { + case ir.TableKindPointer: + rs.bad = true + case ir.TableKindTable: + rs.msgBody(d, depth) + case ir.TableKindArray, ir.TableKindKeyed: + rs.msgElements(d, entry, depth) + case ir.TableKindUnion, ir.TableKindEnum: + rs.msgPayload(d, entry, depth) + case ir.TableKindNoPayload: + // nothing rides, and the outer length is what says so + case ir.TableKindString, ir.TableKindEscape, ir.TableKindWstring: + // a content copied whole carries no inner length: the outer one frames + // it, which is what a keyed slot's element takes + rs.msgText(d, entry) + default: + rs.msgScalar(d, entry) + } +} + +// msgText is a text or blob content under an outer length: the same bytes +// msgPayload writes, without the inner length the outer one replaces. +func (rs *resolver) msgText(d *bitDecoder, entry ir.TableVocabularyEntry) { + b := rs.msgOpaque(d, entry) + if rs.bad { + return + } + rs.out = append(rs.out, b...) +} + +// msgElements is one array's or one keyed body's content: the element kind, the +// count, and the elements. A KEYED SLOT IS A TRIPLE, the key reference, the +// element's own length, and the element, and its keys resolve at every element +// kind (§3.2, §6.6). +func (rs *resolver) msgElements(d *bitDecoder, entry ir.TableVocabularyEntry, depth int) { + shape := entry.Shape + if shape.Elem == ir.TableKindPointer { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + rs.bad = true + return + } + n := uint64(shape.Min) + width := ir.TableMessageCountBits(shape) + if entry.Kind == ir.TableKindKeyed { + width, n = ir.TableMessageBitsRequired(0, shape.Max), 0 + } + if width > 0 { + raw, ok := d.r.get(width) + if !ok { + rs.bad = true + return + } + n = raw + if entry.Kind == ir.TableKindArray { + n += uint64(shape.Min) + } + } + if ir.TableMessageAligns(entry.Kind, shape) && !d.r.align() { + rs.bad = true + return + } + inner := ir.TableMessageShape{} + if shape.Inner != nil { + inner = *shape.Inner + } + element := ir.TableVocabularyEntry{Kind: shape.Elem, Shape: inner} + rs.u8(shape.Elem) + rs.u64(n) + for range n { + if rs.bad { + return + } + if entry.Kind == ir.TableKindKeyed { + rs.u64(rs.msgRef(d)) + rs.msgFramed(d, element, depth) + continue + } + rs.msgPayload(d, element, depth) + } +} + +// msgBody resolves one nested body: its fields, each an entry REFERENCE and the +// payload that entry's shape frames, ending at the body's own ZERO REFERENCE. +func (rs *resolver) msgBody(d *bitDecoder, depth int) { + for { + if rs.bad { + return + } + ref, ok := d.reference() + if !ok { + rs.bad = true + return + } + if ref == 0 { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + rs.u64(0) + return + } + entry, named := d.entry(ref) + if !named || reserved(entry.Id) || entry.Id == 0 { + rs.bad = true + return + } + rs.u64(entry.Id) + rs.u8(entry.Kind) + rs.msgPayload(d, entry, depth) + } +} + +// msgScalar transcodes ONE announced value: read at the width the shape states, +// written at the width the file form spells. It is the third of the three ways +// the form changes a value (§3.3) taken in the one direction retention has: a +// mask rides at its declared W bits and a compressed float as its QUANTIZED +// INDEX here, and the file carries the mask's own storage width and the float +// the index names. +// +// NO CLAMP FIRES AND NO RANGE IS APPLIED. The value is the SENDER's, under an +// id this reader cannot name, so there is no declaration to bound it by: the +// walk reads kind bytes, lengths and references and takes no branch on a +// payload byte (§6.6, THE SECURITY BOUND). +func (rs *resolver) msgScalar(d *bitDecoder, entry ir.TableVocabularyEntry) { + kind := int(entry.Kind) + shape := entry.Shape + switch kind { + case ir.TableKindBool: + v, ok := d.r.get(1) + if !ok { + rs.bad = true + return + } + rs.u8(uint8(v)) + return + case ir.TableKindF64: + raw, ok := d.r.get(64) + if !ok { + rs.bad = true + return + } + rs.out = binary.LittleEndian.AppendUint64(rs.out, raw) + return + case ir.TableKindF32: + rs.msgFloat32(d, shape) + return + } + width := int(ir.TableMessageValueBits(entry.Kind, shape)) + if width < 0 { + rs.bad = true + return + } + base := shape.Base + if base == nil { + base = big.NewInt(0) + } + if ir.TableKindWide(kind) { + var value *big.Int + if shape.Packing == ir.TableMessageRanged { + offset, ok := d.r.getBig(width) + if !ok { + rs.bad = true + return + } + value = new(big.Int).Add(offset, base) + } else { + raw, ok := d.r.bytes(16) + if !ok { + rs.bad = true + return + } + value = tabletext.WideFromBytes(raw, kind) + } + rs.out = append(rs.out, tabletext.WideBytes(value, kind)...) + return + } + raw, ok := d.r.get(width) + if !ok { + rs.bad = true + return + } + signed := ir.TableKindSigned(kind) + var value uint64 + switch { + case shape.Packing == ir.TableMessageRanged && signed: + value = uint64(int64(raw) + base.Int64()) + case shape.Packing == ir.TableMessageRanged: + value = raw + base.Uint64() // the unsigned domain, whole: the sum wraps into the bits below + case signed && width < 64 && width > 0: + shift := uint(64 - width) + value = uint64(int64(raw<> shift) + default: + value = raw + } + rs.msgWord(value, kind) +} + +// msgWord writes one integer at the FILE's own width for its kind, little +// endian, which is two's complement where the kind is signed. +func (rs *resolver) msgWord(value uint64, kind int) { + switch ir.TableKindWidth(kind) { + case 1: + rs.u8(uint8(value)) + case 2: + rs.out = binary.LittleEndian.AppendUint16(rs.out, uint16(value)) + case 4: + rs.out = binary.LittleEndian.AppendUint32(rs.out, uint32(value)) + case 8: + rs.u64(value) + default: + rs.bad = true + } +} + +// msgFloat32 reads an f32 at the sender's packing: the IEEE bit pattern where +// it rides raw, and the float its INDEX names where the sender quantized it +// (SPEC.md §4.3). An index above `count` is rejected, as the packet wire +// rejects it, and drops the record. +func (rs *resolver) msgFloat32(d *bitDecoder, shape ir.TableMessageShape) { + if shape.Packing == ir.TableMessageQuantized { + index, ok := d.r.get(int(shape.Bits)) + if !ok { + rs.bad = true + return + } + count, _, derived := ir.TableMessageQuantization(shape) + if !derived || index > uint64(count) { + rs.bad = true + return + } + v := ir.TableMessageDequantize(shape, uint32(index)) + rs.out = binary.LittleEndian.AppendUint32(rs.out, math.Float32bits(v)) + return + } + raw, ok := d.r.get(32) + if !ok { + rs.bad = true + return + } + rs.out = binary.LittleEndian.AppendUint32(rs.out, uint32(raw)) +} diff --git a/test/conformance/README.md b/test/conformance/README.md index f6bc2e573..9e6316765 100644 --- a/test/conformance/README.md +++ b/test/conformance/README.md @@ -156,6 +156,8 @@ One process per surface, so a runtime starts once rather than once per case. | `wire` | `instance` | Load the wire file, Save, the bytes | the wire golden | | `message` | `message` | AnnounceRead the connection's announcement, LoadMessages the message-form batch against it, SaveMessages, the bytes | the message-form golden | | `report` | `report` | Load the wire file, the report as `u,k,w,c,d,m,verdict\n` | `reports.txt` | +| `retain` | `retain`, `retain-message` | LoadRetain the wire at the row's two capacities, MeasureRetain, SaveRetain, the counters as `,, \n` | the counters in the manifest | +| `retain-save` | the rows that name a save | the same round trip, the SAVED BYTES, one body after another | the pinned saves, back to back | | `json-read` | `instance` | FromJson `json/.json`, Save, the bytes | the wire golden | | `json-write` | `instance` | Load the wire file, ToJson, the text, as `.json` | `json/.json` | | `json-hostile` | `json-hostile` | FromJson `/.json`, the report as `u,k,w,c,d,m,verdict\n`, or `refused\n` | the verdict in the manifest | @@ -187,6 +189,25 @@ which is the wire form's own absence one grain up: a port carries the FILE form alone, and its `LoadMessages`, `MeasureMessages` and `SaveMessages` are the follow-on PORTING.md M20 already registers. +**THE TWO RETAIN SURFACES ARE ONE ROUND TRIP AND TWO SHAPES** +(docs/SPEC-TABLES.md §6.6): `retain` prints the COUNTERS and `retain-save` +writes the BYTES, split for the reason the block surfaces are: a counter cannot +see a record that moved out of a body, and a byte string cannot say which of the +two stores was short. Both run the same pair, so a leg that answers one answers +the other. + +**A ROW'S TWO CAPACITIES ARE RULES A DRIVER APPLIES, and one of them is the +whole reason they are not numbers.** A retained record's byte cost is the port's +own, so `short` means ONE BYTE SHORT OF THE LAST RECORD: the driver loads once +at a roomy capacity, reads what its own buffer used, and loads again one byte +under that. The ID LIST's capacity is a COUNT and travels as one. **The C++ +reference answers both surfaces and the eight ports print ABSENT**, which is the +same absence the `message` and `wire` surfaces already carry one grain up: a +port carries no retention at all, so `LoadRetain`, `MeasureRetain` and +`SaveRetain` are a missing FEATURE and not a failing test, and §6.6's own +backend status says so. A `retain-message` row needs the form-2 read beside +them, which no port has either. + **`cook-write` IS THE ONE SURFACE WHERE A LANGUAGE WRITES AN ACCELERATOR RATHER THAN READING ONE, and the expectation is the TOOL's file.** Every other cook surface asks whether a reader agrees about bytes somebody else produced; this diff --git a/test/conformance/cpp/main.cpp b/test/conformance/cpp/main.cpp index 520682149..73c95a82e 100644 --- a/test/conformance/cpp/main.cpp +++ b/test/conformance/cpp/main.cpp @@ -49,6 +49,10 @@ #include "BackendTable.h" #include "VocabTable.h" #include "Vocab9Table.h" +// the RETAIN-UNKNOWN reader (docs/SPEC-TABLES.md §6.6): the older build, which +// cannot name what RT2 and RT3 wrote. Only the READER is here, the two newer +// builds are writers of the pinned vectors and this driver never runs one. +#include "RT1Table.h" // the POINTERED unit (docs/SPEC-TABLES.md §6.2): a region and a root pointer, // never a value, which is why it gets its own row shape below #include "GraphTable.h" @@ -442,6 +446,164 @@ static const MsgCodec msg_codecs[] = { MSGVARCODEC( "graphdemo", graphdemo, Scene ), }; +// --------------------------------------------------------------------------- +// RETAIN-UNKNOWN's rows (docs/SPEC-TABLES.md §6.6) +// --------------------------------------------------------------------------- +// +// A retain row is a REGION ROUND TRIP and only that: LoadRetain into a region, +// then MeasureRetain and SaveRetain out of that same region. The row's two +// capacities are RULES rather than numbers, `short` is one byte under what +// this port's own full load used, because a record's byte cost is the port's +// own, so the codec below runs the load twice where the row asks for it. +// +// THE MESSAGE ROW IS THE SAME PAIR WITH A FORM-2 LOAD (§3.3): the batch takes +// one region and one retention buffer a body, the announcement carries the id +// table, and the SAVE is the FILE form on every body, because retention writing +// form 2 refuses by name. +struct RetainAnswer +{ + int32_t retained = 0; + int32_t retain_lost = 0; + int32_t unknown = 0; + int32_t save_lost = 0; + std::vector saved; // the bodies' saves, back to back in body order +}; + +struct RetainCodec +{ + const char * unit; + const char * root; + bool message; + bool ( *run )( const uint8_t * announcement, int64_t announcement_bytes, + const uint8_t * wire, int64_t wire_bytes, + bool short_buffer, int64_t id_capacity, RetainAnswer & answer ); +}; + +// the roomy capacity a `full` row takes, the page's own C/8 bound beside it, a +// receiver's room for a LARGER unit's vocabulary (§3.3), and the form's own +// batch bound +static const int64_t kRetainRoomy = 1 << 20; +static const int64_t kRetainMessageEntries = 256; +static const int64_t kRetainBatchCap = 256; + +#define RETAINCODEC( unit_key, ns, type ) \ + { \ + unit_key, #type, false, \ + []( const uint8_t *, int64_t, const uint8_t * wire, int64_t n, \ + bool short_buffer, int64_t id_capacity, RetainAnswer & answer ) { \ + std::vector bytes( (size_t) kRetainRoomy, 0 ); \ + std::vector ids( \ + (size_t) ( id_capacity < 0 ? kRetainRoomy / 8 : id_capacity ) ); \ + int64_t need = ns::type##LoadMeasure( wire, n ); \ + if ( need < 0 ) return false; \ + int64_t capacity = kRetainRoomy; \ + if ( short_buffer ) \ + { \ + /* ONE BYTE SHORT OF THE LAST RECORD, which is this port's own */ \ + /* full load less one: a record's byte cost is the port's (§6.6) */ \ + std::vector probe_region( (size_t) need, 0 ); \ + ns::TableRetain probe; \ + probe.bytes = bytes.data(); \ + probe.capacity = kRetainRoomy; \ + probe.ids = ids.data(); \ + probe.id_capacity = (int32_t) ids.size(); \ + ns::TableReport ignored; \ + if ( ns::type##LoadRetain( probe_region.data(), need, wire, n, &probe, &ignored ) == NULL ) return false; \ + capacity = probe.used - 1; \ + } \ + std::vector region( (size_t) need, 0 ); \ + ns::TableRetain retain; \ + retain.bytes = bytes.data(); \ + retain.capacity = capacity; \ + retain.ids = ids.data(); \ + retain.id_capacity = (int32_t) ids.size(); \ + ns::TableReport load; \ + const ns::type * root = ns::type##LoadRetain( region.data(), need, wire, n, &retain, &load ); \ + if ( root == NULL ) return false; \ + answer.retained = load.retained; \ + answer.retain_lost = load.retain_lost; \ + answer.unknown = load.unknown; \ + ns::TableReport save; \ + int64_t size = ns::type##MeasureRetain( root, &retain ); \ + if ( size < 0 ) return false; \ + answer.saved.assign( (size_t) size, 0 ); \ + if ( ns::type##SaveRetain( root, &retain, answer.saved.data(), size, &save ) != size ) return false; \ + answer.save_lost = save.retain_lost; \ + return true; \ + } \ + } + +#define RETAINMSGCODEC( unit_key, ns, type ) \ + { \ + unit_key, #type, true, \ + []( const uint8_t * a, int64_t an, const uint8_t * wire, int64_t n, \ + bool short_buffer, int64_t id_capacity, RetainAnswer & answer ) { \ + std::vector entries( (size_t) kRetainMessageEntries ); \ + ns::TableVocabulary vocabulary( entries.data(), kRetainMessageEntries ); \ + ns::TableReport open; \ + if ( !ns::AnnounceRead( vocabulary, a, an, &open ) ) return false; \ + int64_t need = ns::type##LoadMeasure( vocabulary, wire, n ); \ + if ( need < 0 ) return false; \ + /* THE BATCH'S OWN COUNT sizes the caller's storage: one root, one */ \ + /* retention buffer and one id list a BODY (docs/SPEC-TABLES.md §3.3) */ \ + ns::TableMessageBatchReader br; \ + ns::TableReport counting; \ + const int64_t bodies = ns::TableMessageBatchOpen( br, vocabulary, wire, n, &counting ); \ + if ( bodies < 0 || bodies > kRetainBatchCap ) return false; \ + std::vector roots( (size_t) bodies, NULL ); \ + std::vector > stores( (size_t) bodies ); \ + std::vector > lists( (size_t) bodies ); \ + std::vector retains( (size_t) bodies ); \ + for ( size_t i = 0; i < retains.size(); i++ ) \ + { \ + stores[i].assign( (size_t) kRetainRoomy, 0 ); \ + lists[i].resize( (size_t) ( id_capacity < 0 ? kRetainRoomy / 8 : id_capacity ) ); \ + retains[i].bytes = stores[i].data(); \ + retains[i].capacity = kRetainRoomy; \ + retains[i].ids = lists[i].data(); \ + retains[i].id_capacity = (int32_t) lists[i].size(); \ + } \ + (void) short_buffer; /* no message row asks for the short rule yet */ \ + std::vector region( (size_t) need + 16, 0 ); \ + uint8_t * base = (uint8_t *) ( ( (uintptr_t) region.data() + 15 ) & ~(uintptr_t) 15 );\ + int64_t count = (int64_t) roots.size(); \ + ns::TableReport load; \ + if ( !ns::type##LoadRetainMessages( roots.data(), &count, base, need, vocabulary, \ + wire, n, retains.data(), &load ) ) return false; \ + answer.retained = load.retained; \ + answer.retain_lost = load.retain_lost; \ + answer.unknown = load.unknown; \ + ns::TableReport save; \ + for ( int64_t b = 0; b < count; b++ ) \ + { \ + int64_t size = ns::type##MeasureRetain( roots[b], &retains[b] ); \ + if ( size < 0 ) return false; \ + std::vector one( (size_t) size, 0 ); \ + if ( ns::type##SaveRetain( roots[b], &retains[b], one.data(), size, &save ) != size ) return false; \ + answer.saved.insert( answer.saved.end(), one.begin(), one.end() ); \ + } \ + answer.save_lost = save.retain_lost; \ + return true; \ + } \ + } + +static const RetainCodec retain_codecs[] = { + RETAINCODEC( "tblrt1", tblrt1, Node ), + RETAINMSGCODEC( "tblrt1", tblrt1, Node ), +}; + +static const RetainCodec * find_retain_codec( const std::string & unit, const std::string & root, bool message ) +{ + for ( size_t i = 0; i < sizeof( retain_codecs ) / sizeof( retain_codecs[0] ); i++ ) + { + if ( unit == retain_codecs[i].unit && root == retain_codecs[i].root && message == retain_codecs[i].message ) + { + return &retain_codecs[i]; + } + } + return NULL; +} + static const MsgCodec * find_msg_codec( const std::string & unit, const std::string & root ) { for ( size_t i = 0; i < sizeof( msg_codecs ) / sizeof( msg_codecs[0] ); i++ ) @@ -832,6 +994,69 @@ static int surface_message( const std::string & out ) return 0; } +// THE RETAIN SURFACES (docs/SPEC-TABLES.md §6.6): the COUNTERS on one and the +// BYTES on the other, so each is one shape. A row's `retain` line carries the +// wire and the two capacities; nothing about the answer reaches the driver. +static bool retain_row( const std::vector & f, RetainAnswer & answer ) +{ + // retain + // retain-message + const bool message = f[0] == "retain-message"; + const size_t at = message ? 1 : 0; + const std::string unit = f[2 + at], root = f[3 + at], wire_path = f[4 + at]; + const bool short_buffer = f[5 + at] == "short"; + int64_t ids = -1; + if ( f[6 + at] != "full" ) { ids = (int64_t) strtoll( f[6 + at].c_str(), NULL, 10 ); } + + std::vector announcement; + if ( message ) + { + std::string path; + for ( size_t j = 0; j < manifest_lines.size(); j++ ) + { + const std::vector & c = manifest_lines[j].field; + if ( c[0] == "connection" && c[1] == f[2] ) { path = c[4]; break; } + } + if ( path.empty() ) { fprintf( stderr, "driver: no connection %s\n", f[2].c_str() ); return false; } + if ( !slurp( path.c_str(), announcement ) ) { fprintf( stderr, "driver: cannot read %s\n", path.c_str() ); return false; } + } + std::vector wire; + if ( !slurp( wire_path.c_str(), wire ) ) { fprintf( stderr, "driver: cannot read %s\n", wire_path.c_str() ); return false; } + const RetainCodec * codec = find_retain_codec( unit, root, message ); + if ( codec == NULL ) { fprintf( stderr, "driver: no retain codec for %s.%s\n", unit.c_str(), root.c_str() ); return false; } + return codec->run( announcement.data(), (int64_t) announcement.size(), + wire.data(), (int64_t) wire.size(), short_buffer, ids, answer ); +} + +static int surface_retain( const std::string & out ) +{ + for ( size_t i = 0; i < manifest_lines.size(); i++ ) + { + const std::vector & f = manifest_lines[i].field; + if ( f[0] != "retain" && f[0] != "retain-message" ) continue; + RetainAnswer answer; + if ( !retain_row( f, answer ) ) return 1; + char text[128]; + int n = snprintf( text, sizeof( text ), "%d,%d,%d %d\n", + answer.retained, answer.retain_lost, answer.unknown, answer.save_lost ); + if ( !spill( out, f[1], text, (size_t) n ) ) return 1; + } + return 0; +} + +static int surface_retain_save( const std::string & out ) +{ + for ( size_t i = 0; i < manifest_lines.size(); i++ ) + { + const std::vector & f = manifest_lines[i].field; + if ( f[0] != "retain" && f[0] != "retain-message" ) continue; + RetainAnswer answer; + if ( !retain_row( f, answer ) ) return 1; + if ( !spill( out, f[1], answer.saved.data(), answer.saved.size() ) ) return 1; + } + return 0; +} + static int surface_report( const std::string & out ) { for ( size_t i = 0; i < manifest_lines.size(); i++ ) @@ -1296,7 +1521,7 @@ int main( int argc, char ** argv ) const std::string surface = argv[2]; if ( surface == "list" ) { - printf( "wire\nmessage\nreport\njson-read\njson-write\njson-hostile\ncook-write\nblock\nblock-foreign\nblock-dump\nforgery\nblock-reason\n" ); + printf( "wire\nmessage\nreport\nretain\nretain-save\njson-read\njson-write\njson-hostile\ncook-write\nblock\nblock-foreign\nblock-dump\nforgery\nblock-reason\n" ); return 0; } if ( argc < 4 ) @@ -1309,6 +1534,8 @@ int main( int argc, char ** argv ) if ( surface == "wire" ) return surface_wire( out ); if ( surface == "message" ) return surface_message( out ); if ( surface == "report" ) return surface_report( out ); + if ( surface == "retain" ) return surface_retain( out ); + if ( surface == "retain-save" ) return surface_retain_save( out ); if ( surface == "json-read" ) return surface_json_read( out ); if ( surface == "json-write" ) return surface_json_write( out ); if ( surface == "json-hostile" ) return surface_json_hostile( out ); diff --git a/test/conformance/harness/manifest.go b/test/conformance/harness/manifest.go index 8b085aca2..4eacc5fcc 100644 --- a/test/conformance/harness/manifest.go +++ b/test/conformance/harness/manifest.go @@ -265,11 +265,69 @@ type Message struct { MessageWire string } +// RetainCase is one RETAIN-UNKNOWN row (docs/SPEC-TABLES.md §6.6): a wire +// loaded with retention, the counters that load owes, and the file the pair +// writes back. +// +// THE TWO CAPACITIES ARE STATED AS RULES AND NOT AS NUMBERS where a number +// would not travel. A record's BYTE cost is the port's own, so `Short` says ONE +// BYTE SHORT OF THE LAST RECORD, a leg loads once at a capacity the whole load +// fits in, reads what it used, and loads again one byte under that, and two +// layouts then drop the same record rather than different ones. The ID LIST's +// capacity is a COUNT and travels as one, so `Ids` is either full or the +// entries the caller's list holds. +// +// A MESSAGE row is the form-2 one (§3.3): the wire is a BATCH, `Connection` +// names the announcement its references resolve against, and the SAVES are the +// FILE form, one a body, back to back in body order, which is the one +// direction retention crosses the forms, because a form-2 SaveRetain refuses by +// name. +type RetainCase struct { + Name string + Unit string // the READER's unit; empty on a message row, which names a connection + Connection string // the announcement a message row resolves against + Root string + Wire string + Short bool // the retention buffer is one byte short of the last record + Ids int // the caller's id-list entries; -1 is full + Load RetainCounts + SaveLost int // the `retain_lost` the SAVE adds + Saves []string // the pinned saves, in body order; empty is a row that does not save + Message bool +} + +// RetainCounts is a retaining load's own three: the two retention counters and +// the `unknown` they ride beside, which retention never moves. +type RetainCounts struct{ Retained, RetainLost, Unknown int } + +func (c RetainCounts) String() string { + return fmt.Sprintf("%d,%d,%d", c.Retained, c.RetainLost, c.Unknown) +} + +// ParseRetainCounts reads `,,`. +func ParseRetainCounts(text string) (RetainCounts, error) { + parts := strings.Split(text, ",") + if len(parts) != 3 { + return RetainCounts{}, fmt.Errorf("%q is not retained,retain_lost,unknown", text) + } + var c RetainCounts + into := []*int{&c.Retained, &c.RetainLost, &c.Unknown} + for i, p := range parts { + v, err := strconv.Atoi(p) + if err != nil { + return RetainCounts{}, fmt.Errorf("%q is not a count in %q", p, text) + } + *into[i] = v + } + return c, nil +} + // Manifest is the whole registry. type Manifest struct { Units []Unit Connections []Connection Messages []Message + Retains []RetainCase Instances []Instance Reports []ReportCase Hostiles []Hostile @@ -370,6 +428,57 @@ func ReadManifest(path, jsonDir string) (*Manifest, error) { return nil, fmt.Errorf("%s: message takes name, connection, root, file-form wire, message-form wire", where) } m.Messages = append(m.Messages, Message{Name: f[1], Connection: f[2], Root: f[3], FileWire: f[4], MessageWire: f[5]}) + case "retain", "retain-message": + // retain + // retain-message ... + rc := RetainCase{Name: f[1], Message: f[0] == "retain-message"} + want := 10 + if rc.Message { + want = 11 + } + if len(f) != want { + return nil, fmt.Errorf("%s: %s takes %d fields and this line has %d", where, f[0], want, len(f)) + } + // A MESSAGE ROW NAMES BOTH PEERS: the CONNECTION whose announcement + // its references resolve against, which is the sender's, and the + // READER's own unit, which is the build that cannot name what the + // sender wrote (docs/SPEC-TABLES.md §3.3, §6.6). + if rc.Message { + rc.Connection, rc.Unit = f[2], f[3] + f = append(f[:2], f[3:]...) // the columns below line up with the file row's + } else { + rc.Unit = f[2] + } + rc.Root, rc.Wire = f[3], f[4] + switch f[5] { + case "full": + case "short": + rc.Short = true + default: + return nil, fmt.Errorf("%s: %q is not a retention capacity; it is full or short", where, f[5]) + } + rc.Ids = -1 + if f[6] != "full" { + n, err := strconv.Atoi(f[6]) + if err != nil || n < 0 { + return nil, fmt.Errorf("%s: %q is not an id-list capacity", where, f[6]) + } + rc.Ids = n + } + counts, err := ParseRetainCounts(f[7]) + if err != nil { + return nil, fmt.Errorf("%s: %w", where, err) + } + rc.Load = counts + lost, err := strconv.Atoi(f[8]) + if err != nil || lost < 0 { + return nil, fmt.Errorf("%s: %q is not the save's own retain_lost", where, f[8]) + } + rc.SaveLost = lost + if f[9] != "-" { + rc.Saves = strings.Split(f[9], ",") + } + m.Retains = append(m.Retains, rc) case "report": if len(f) != 5 { return nil, fmt.Errorf("%s: report takes case, unit, root, wire", where) diff --git a/test/conformance/harness/message_test.go b/test/conformance/harness/message_test.go index aacd0bbb6..d46527ed6 100644 --- a/test/conformance/harness/message_test.go +++ b/test/conformance/harness/message_test.go @@ -170,8 +170,16 @@ func TestTheTailIsUnconditional(t *testing.T) { entries := ir.TableVocabulary(unit) var tables []uint64 for name := range ir.TableClosure(unit) { + // A MAP'S GENERATED ENTRY IS NOT A TABLE OF THE DECLARATION'S and + // its name is on no wire, so the tail never names it (§2.8, §20.2). + // The vocabulary skips it by `MapEntryOf`, and so does this: no + // connection unit carried a map until the retain rows landed one. + if st := unit.Tables[name]; st != nil && st.MapEntryOf != "" { + continue + } if unit.Tables[name] != nil { - tables = append(tables, ir.TableWireId(name)) + // the id is the table's WIRE name, so a `was` rename moves no slot + tables = append(tables, ir.TableWireId(unit.Tables[name].WireName())) } } // the tail is the last 4 + one-per-table entries, in the fixed order diff --git a/test/conformance/harness/retain_test.go b/test/conformance/harness/retain_test.go new file mode 100644 index 000000000..06d932af7 --- /dev/null +++ b/test/conformance/harness/retain_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "bytes" + "os" + "testing" + + "github.com/mas-bandwidth/schema/v2/internal/tabletext" + "github.com/mas-bandwidth/schema/v2/internal/tablewire" +) + +// THE ORACLE CLAIMS THE RETAIN SURFACE (docs/SPEC-TABLES.md §6.6). The +// compiler's engine is a third reading of §3 and §6.6, written from the page +// rather than from a backend, and this row drives it through EVERY `retain` and +// `retain-message` line of the manifest: the same wires, the same two +// capacities, and the same counters and saved bytes the C++ reference answers +// on the same rows. +// +// It is the manifest's own expectation that is checked here and not a second +// one. A row whose counters drift shows up in two places at once, here and in +// the matrix, which is what makes the manifest data rather than a restatement +// of what one engine happens to do. +func TestTheOracleAnswersEveryRetainRow(t *testing.T) { + m, _, u := corpus(t) + if len(m.Retains) == 0 { + t.Fatal("the manifest names no retain row") + } + for _, rc := range m.Retains { + t.Run(rc.Name, func(t *testing.T) { + // A MESSAGE ROW READS TWO FILES, because a batch's id table is + // somewhere else (§3.3): the CONNECTION's announcement carries it. + unitKey := rc.Unit + var vocabulary *tablewire.Vocabulary + if rc.Message { + c, err := m.LookupConnection(rc.Connection) + if err != nil { + t.Fatal(err) + } + // the announcement is the SENDER's and the row's unit column + // is the READER's, which is the build that cannot name what + // the sender wrote + _, vocabulary = announced(t, u, c) + } + unit, err := u.get(unitKey) + if err != nil { + t.Fatal(err) + } + model := tabletext.NewModel(unit) + def := model.Lookup(rc.Root) + if def == nil { + t.Fatalf("%s declares no root %s", unitKey, rc.Root) + } + wire := wireBytes(t, rc.Wire) + + load := func(capacity, ids int) ([]*tabletext.Instance, []*tablewire.Retain, tabletext.Report) { + t.Helper() + var report tabletext.Report + if !rc.Message { + inst := model.New(def) + store := &tablewire.Retain{Capacity: capacity, IdCapacity: ids} + ok, err := tablewire.DecodeRetain(model, inst, wire, store, &report) + if err != nil || !ok || report.Malformed { + t.Fatalf("the load reported damage on a sound wire: ok=%v err=%v %+v", ok, err, report) + } + return []*tabletext.Instance{inst}, []*tablewire.Retain{store}, report + } + count, err := tablewire.MessageCount(wire, vocabulary) + if err != nil { + t.Fatal(err) + } + insts := make([]*tabletext.Instance, count) + stores := make([]*tablewire.Retain, count) + for i := range insts { + insts[i] = model.New(def) + stores[i] = &tablewire.Retain{Capacity: capacity, IdCapacity: ids} + } + read, ok, err := tablewire.DecodeRetainMessages(model, insts, wire, vocabulary, stores, &report) + if err != nil || !ok || report.Malformed || read != count { + t.Fatalf("the load reported damage on a sound batch: read=%d ok=%v err=%v %+v", read, ok, err, report) + } + return insts, stores, report + } + + // THE CAPACITY RULES, applied rather than read as numbers: a + // record's byte cost is the port's own, so `short` is one byte + // under what THIS engine's own full load used (§6.6). + const roomy = 1 << 20 + ids := rc.Ids + if ids < 0 { + ids = roomy / 8 // the page's own C/8 bound, which no file reaches + } + capacity := roomy + if rc.Short { + _, full, _ := load(roomy, roomy/8) + used := 0 + for _, store := range full { + used += store.Used() + } + capacity = used - 1 + } + insts, stores, report := load(capacity, ids) + + got := RetainCounts{Retained: report.Retained, RetainLost: report.RetainLost, Unknown: report.Unknown} + if got != rc.Load { + t.Fatalf("the load says %s and the manifest says %s", got, rc.Load) + } + if report.KindMismatch != 0 || report.Clamped != 0 || report.Widened != 0 || report.Duplicate != 0 { + t.Fatalf("retention moved a read counter: %+v", report) + } + + // AND THE SAVE, which is the FILE form on every row: retention + // writing form 2 refuses by name (§3.3). + var save tabletext.Report + var saved []byte + for i, inst := range insts { + out, err := tablewire.EncodeRetain(model, inst, stores[i], &save) + if err != nil { + t.Fatal(err) + } + saved = append(saved, out...) + } + if save.RetainLost != rc.SaveLost { + t.Fatalf("the save lost %d and the manifest says %d", save.RetainLost, rc.SaveLost) + } + if len(rc.Saves) == 0 { + return + } + var want []byte + for _, path := range rc.Saves { + bs, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + want = append(want, bs...) + } + if !bytes.Equal(saved, want) { + t.Fatalf("the save is %d bytes and the pin is %d, and they differ at byte %d", + len(saved), len(want), firstDifference(saved, want)) + } + }) + } +} diff --git a/test/conformance/harness/run.go b/test/conformance/harness/run.go index e6631ec3f..92ce17275 100644 --- a/test/conformance/harness/run.go +++ b/test/conformance/harness/run.go @@ -22,7 +22,7 @@ import ( ) // surfaces, in the order the matrix prints them. -var surfaces = []string{"wire", "message", "report", "json-read", "json-write", "json-hostile", +var surfaces = []string{"wire", "message", "report", "retain", "retain-save", "json-read", "json-write", "json-hostile", "cook", "cook-write", "cook-foreign", "block", "block-foreign", "block-dump", "forgery", "cook-forgery", "cook-reason", "block-reason"} @@ -161,6 +161,25 @@ func deriveManifest(m *Manifest, path string) error { for _, msg := range m.Messages { fmt.Fprintf(&b, "message %s %s %s %s %s\n", msg.Name, msg.Connection, msg.Root, msg.FileWire, msg.MessageWire) } + for _, rc := range m.Retains { + // THE ANSWER IS WITHHELD, as it is everywhere else: a driver is handed + // the wire and the two capacities and nothing about the counters or the + // bytes it owes (docs/SPEC-TABLES.md §6.6). `short` and `full` travel as + // the RULES they are, because a record's byte cost is the port's own. + kind, who := "retain", rc.Unit + if rc.Message { + kind, who = "retain-message", rc.Connection+" "+rc.Unit + } + ids := "full" + if rc.Ids >= 0 { + ids = strconv.Itoa(rc.Ids) + } + capacity := "full" + if rc.Short { + capacity = "short" + } + fmt.Fprintf(&b, "%s %s %s %s %s %s %s\n", kind, rc.Name, who, rc.Root, rc.Wire, capacity, ids) + } for _, r := range m.Reports { fmt.Fprintf(&b, "report %s %s %s %s\n", r.Name, r.Unit, r.Root, r.Wire) } @@ -228,6 +247,34 @@ func expectations(m *Manifest, surface string, reports map[string]Counts, jsonDi } out = append(out, expectation{msg.Name, want}) } + case "retain": + // RETAIN-UNKNOWN's COUNTERS (docs/SPEC-TABLES.md §6.6): the load's two + // beside the `unknown` retention never moves, and the save's own + // `retain_lost` behind them. The bytes are the other surface's, so each + // of the two is one shape. + for _, rc := range m.Retains { + out = append(out, expectation{rc.Name, []byte(rc.Load.String() + " " + strconv.Itoa(rc.SaveLost) + "\n")}) + } + case "retain-save": + // AND THE BYTES THE PAIR WRITES BACK, which a counter cannot see: a + // dropped record moves a field out of a body and every reference behind + // it. A row that does not save is not a case here. A MESSAGE row's + // saves are FILES, one a body, back to back in body order, because a + // form-2 SaveRetain refuses by name (§3.3). + for _, rc := range m.Retains { + if len(rc.Saves) == 0 { + continue + } + var want []byte + for _, path := range rc.Saves { + bytes, err := os.ReadFile(path) + if err != nil { + return nil, err + } + want = append(want, bytes...) + } + out = append(out, expectation{rc.Name, want}) + } case "json-write": for _, i := range m.Instances { if i.NoText { diff --git a/test/tables/retain_main.cpp b/test/tables/retain_main.cpp index 13382426f..2fa365964 100644 --- a/test/tables/retain_main.cpp +++ b/test/tables/retain_main.cpp @@ -891,6 +891,204 @@ static void walk_is_linear() } } +// ---- THE MESSAGE FORM'S LoadRetain (docs/SPEC-TABLES.md §3.3, §6.6) ---- +// +// RETENTION CROSSES THE FORMS IN ONE DIRECTION. `LoadRetain` reads a form-2 +// body as it reads a file's, the resolving walk replacing every reference with +// the id it names against the CONNECTION'S VOCABULARY instead of a trailer, +// and `SaveRetain` writing form 2 refuses by name. So the round trip a caller +// has is form 2 in and form 1 out: the file carries its own table and takes +// §6.6 unchanged. +// +// The batch below is RT2's, over the same values the file rows read, and it +// carries every class this form can put on the unknown arm: the fields at +// three depths that are retained, the whole `parcel` table the resolving walk +// reads, a field of kind 17, an array whose element kind is 17, a table whose +// payload meets a 17 three bodies down, an unknown enum variant reference, an +// unknown union arm id and an unknown keyed-array slot. + +// fill_rt2_message fills one RT2 root for the batch. `full` is the body that +// carries every class; the other is the batch's second body, which carries one +// unknown field and nothing else, so that a record of the first body can never +// be placed in the second. +static void fill_rt2_message( tblrt2::NodeBuilder & builder, bool full ) +{ + tblrt2::Node * root = builder.GetRoot(); + memcpy( root->name, full ? "world" : "again", 6 ); + root->name_length = 5; + root->extra = full ? 11 : 12; + root->inner.hits = 3; + root->inner.future = 21; + memcpy( root->inner.tag, "in", 3 ); + root->inner.tag_length = 2; + if ( !full ) { return; } + + root->tier = tblrt2::Slot::Extra; // an unknown enum VARIANT reference + + root->items_count = 2; + root->items[0].hits = 1; root->items[0].future = 31; + root->items[1].hits = 2; root->items[1].future = 32; + + root->banks.slots[0].hits = 4; root->banks.slots[0].future = 41; + root->banks.slots[2].hits = 6; root->banks.slots[2].future = 43; // an unknown keyed SLOT + + root->pick.type = tblrt2::PickType::Gamma; // an unknown union ARM id + root->pick.gamma = 55; + + tblrt2::TableSlot leaf = builder.Alloc(); + leaf->value = 77; + root->head = leaf; + + // THE NODE-INDEX CLASSES, each on a field RT1 cannot name: kind 17 itself, + // an array whose element kind is 17, and a table whose payload meets a 17 + // three bodies down + tblrt2::TableSlot ghost = builder.Alloc(); + ghost->value = 78; + root->ghost = ghost; + root->ghosts_count = 1; + root->ghosts[0] = ghost; + root->outer.plain = 79; + root->outer.mid.deeper.link = ghost; + + tblrt2::Inner * one = tblrt2::NodeEntriesInsert( builder.main, root->entries, "k1" ); + if ( one != NULL ) { one->hits = 7; one->future = 61; } + + tblrt2::Inner * item = tblrt2::NodeListAdd( builder.main, root->list ); + if ( item != NULL ) { item->hits = 8; item->future = 71; } + + root->parcel.grade = tblrt2::Grade::Silver; // kind 30, a variant reference + root->parcel.fit.type = tblrt2::FittingType::Bolt; // kind 15, a table arm + tblrt2::BoltReset( root->parcel.fit.bolt ); + root->parcel.fit.bolt.weight = 101; + root->parcel.bins.slots[0].weight = 102; // kind 16, keyed by Grade + root->parcel.bins.slots[1].weight = 103; + root->parcel.core.weight = 104; // kind 13, a nested body + root->parcel.stack_count = 2; // kind 14, table elements + root->parcel.stack[0].weight = 105; + root->parcel.stack[1].weight = 106; +} + +static void message_form() +{ + tblrt2::NodeBuilder first; + tblrt2::NodeBuilder second; + fill_rt2_message( first, true ); + fill_rt2_message( second, false ); + CHECK( first.Lock() ); + CHECK( second.Lock() ); + const tblrt2::Node * roots[2] = { first.AsConst(), second.AsConst() }; + + tblrt2::TableReport wrote; + uint8_t batch[ 8192 ]; + const int64_t n = tblrt2::NodeSaveMessages( roots, 2, batch, sizeof( batch ), &wrote ); + CHECK( n > 0 ); + // THE INPUT IS A PINNED VECTOR, so the oracle reads the same bytes this row + // reads (docs/SPEC-TABLES.md §4.2). It is a MESSAGE and not a file: its + // references resolve against RT2's announced vocabulary. + pin_golden( "retain_message", batch, n ); + + // THE CONNECTION'S VOCABULARY, which is what the resolving walk resolves + // against here where a file has a trailer (§3.3). RT1 declares room for + // RT2's entries, because a receiver meeting a LARGER unit declares more + // than its own count. + uint8_t announcement[ 8192 ]; + const int64_t a = tblrt2::Announce( announcement, sizeof( announcement ) ); + CHECK( a > 0 ); + // THE CONNECTION IS A PINNED VECTOR TOO, because the conformance manifest + // names it: an announcement is an ordinary form-1 file, and the retain rows + // that read a batch resolve against this one (testdata/conformance/tables). + pin_golden( "retain_conn", announcement, a ); + printf( "retain: RT2's build version is 0x%016llx\n", (unsigned long long) tblrt2::BuildVersion ); + static tblrt1::TableMessageEntry entries[ 128 ]; + tblrt1::TableVocabulary vocabulary( entries, 128 ); + tblrt1::TableReport announced; + CHECK( tblrt1::AnnounceRead( vocabulary, announcement, a, &announced ) ); + + // A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): + // each body carries its OWN node directory inside that one region, so a + // record's first step is an index into the directory of the body it came + // from, and the buffer that holds it is that body's own. + Region region; + region.size( tblrt1::NodeLoadMeasure( vocabulary, batch, n ) ); + uint8_t storage[ 2 ][ 8192 ]; + tblrt1::TableRetain::Id ids[ 2 ][ 1024 ]; + tblrt1::TableRetain retains[ 2 ]; + for ( int i = 0; i < 2; i++ ) + { + retains[i].bytes = storage[i]; + retains[i].capacity = (int64_t) sizeof( storage[i] ); + retains[i].ids = ids[i]; + retains[i].id_capacity = 1024; + } + + const tblrt1::Node * read[ 2 ] = { NULL, NULL }; + int64_t count = 2; + tblrt1::TableReport report; + const long long before = allocations; + CHECK( tblrt1::NodeLoadRetainMessages( read, &count, region.base, region.bytes, + vocabulary, batch, n, retains, &report ) ); + CHECK( count == 2 ); + CHECK( read[0] != NULL && read[1] != NULL ); + CHECK( !report.malformed ); + CHECK( allocations == before ); // RETENTION ALLOCATES NOTHING (§6.6) + + // THE FIRST BODY carries eight retained fields, `extra`, `future` at four + // more depths, the map entry's, the list element's, and `parcel`, the whole + // table the resolving walk reads, and six unknowns of the excluded + // classes: the enum variant `tier` names, the union arm `pick` names, the + // keyed slot RT2's third variant writes, a field of kind 17, an array whose + // element kind is 17, and a table whose payload meets a 17 three bodies + // down. THE SECOND BODY carries two retained fields and nothing excluded. + CHECK( report.retained == 10 ); + CHECK( report.retain_lost == 6 ); + CHECK( report.unknown == 17 ); + // and the READ'S OWN COUNTERS ARE UNMOVED: retention can lose a field and + // can never turn a good read into a bad one (§6.6) + CHECK( report.kind_mismatch == 0 && report.clamped == 0 && report.widened == 0 ); + + // AND THE SAVE IS THE FILE FORM'S, because form 2 refuses to write a + // retained record by name (§3.3): the file carries its own table and takes + // §6.6 unchanged. + for ( int i = 0; i < 2; i++ ) + { + tblrt1::TableReport save_report; + const int64_t m = tblrt1::NodeMeasureRetain( read[i], &retains[i] ); + CHECK( m > 0 ); + uint8_t out[ 8192 ]; + const int64_t w = tblrt1::NodeSaveRetain( read[i], &retains[i], out, m, &save_report ); + CHECK( w == m ); // MEASURE AND SAVE DROP THE SAME RECORDS UNDER THE SAME WALK + CHECK( save_report.retain_lost == 0 ); + pin_golden( i == 0 ? "retain_message_save_0" : "retain_message_save_1", out, w ); + + // AND RT2 READS ITS OWN FIELDS BACK out of the file RT1 wrote, which is + // the resolution half read back: every reference the walk resolved is + // re-interned into a trailer whose order is this file's own. + Region back; + back.size( tblrt2::NodeLoadMeasure( out, w ) ); + tblrt2::TableReport read_report; + const tblrt2::Node * again = tblrt2::NodeLoad( back.base, back.bytes, out, w, &read_report ); + CHECK( again != NULL && !read_report.malformed ); + CHECK( again->extra == ( i == 0 ? 11 : 12 ) ); + CHECK( again->inner.future == 21 ); + if ( i == 0 ) + { + CHECK( again->parcel.core.weight == 104 ); + CHECK( again->parcel.grade == tblrt2::Grade::Silver ); + CHECK( again->parcel.fit.type == tblrt2::FittingType::Bolt ); + CHECK( again->parcel.fit.bolt.weight == 101 ); + CHECK( again->parcel.bins.slots[1].weight == 103 ); + CHECK( again->parcel.stack_count == 2 && again->parcel.stack[1].weight == 106 ); + CHECK( again->items[1].future == 32 ); + CHECK( again->banks.slots[0].future == 41 ); + // AND THE EXCLUDED CLASSES ARE GONE, each having counted one + // retain_lost above: nothing put a node index back + CHECK( again->ghost.null() ); + CHECK( again->ghosts_count == 0 ); + CHECK( again->outer.plain == 0 ); + } + } +} + // ---- THE TRAILER IS PERMUTED, AND EVERY REFERENCE MOVES WITH IT ---- // // Moving a field changes the order ids are first used in, and the id table is @@ -936,6 +1134,7 @@ int main( int argc, char ** argv ) disjoint_occurrences(); walk_is_linear(); trailer_is_permuted(); + message_form(); if ( failures != 0 ) { printf( "retain: %d failure(s)\n", failures ); diff --git a/testdata/conformance/tables/FORMAT.md b/testdata/conformance/tables/FORMAT.md index 3cb847350..dc1a1915f 100644 --- a/testdata/conformance/tables/FORMAT.md +++ b/testdata/conformance/tables/FORMAT.md @@ -38,6 +38,8 @@ unit ... instance [no-text] connection message +retain +retain-message report json-hostile cook @@ -75,6 +77,35 @@ refusal every reference is replaced by the id it names and every length recomputed, and their reference bytes are expected to differ, because a file's slots are its own first-use order and a connection's are the unit's projection order. +- **`retain` and `retain-message`** are RETAIN-UNKNOWN's rows + (docs/SPEC-TABLES.md §6.6): a wire loaded with retention on, the counters that + load owes, and the file the pair writes back. `` is the READER's, which + is the build that cannot name what the wire's writer wrote, and `` is + the table it loads into. + + **THE TWO CAPACITIES ARE STATED AS RULES AND NOT AS NUMBERS where a number + would not travel.** A retained record's byte cost is the PORT's own, the + page fixes what a record must carry and leaves the layout to the + implementation, so `` is `full`, a buffer the whole load fits in, + or `short`, ONE BYTE SHORT OF THE LAST RECORD: a leg loads once at a roomy + capacity, reads what it used, and loads again one byte under that. Two + layouts then drop the same record rather than different ones. `` is the + other store, and it IS comparable, because the id list's capacity is a COUNT: + `full`, or the entries the caller's list holds. + + `` is the LOAD's own three, `,,`, + and `` is the `retain_lost` the SAVE adds, read after it: the + report accumulates across the pair and the sum is what the safety check + wants. `` is the pinned save, or `-` where the row's whole expectation + is its counters; a row that saves more than one file lists them + comma-separated, in body order. + + **`retain-message` IS THE FORM-2 ROW** (§3.3): the wire is a BATCH, the + `` names the announcement its references resolve against, and the + SAVES are the FILE form, one a body, which is the one direction retention + crosses the forms, because `SaveRetain` writing form 2 refuses by name. A + batch takes one region and one retention buffer a body, so the saves are as + many files as the batch has bodies. - **`report`** is bytes read by a type that did not write them — the evolution class. The counters and the verdict live in `reports.txt`, keyed by ``. - **`json-hostile`** is one tree per rule the text form states (§16.2, §16.3, diff --git a/testdata/conformance/tables/MANIFEST.txt b/testdata/conformance/tables/MANIFEST.txt index a72a07647..bb1c6ceaa 100644 --- a/testdata/conformance/tables/MANIFEST.txt +++ b/testdata/conformance/tables/MANIFEST.txt @@ -15,6 +15,17 @@ # testdata/conformance/tables/json/.json (§16). The same # instance both ways. # +# retain +# retain-message ... +# one RETAIN-UNKNOWN round trip (§6.6): the wire, the two capacities +# the caller declares, the counters the LOAD owes and the one the +# SAVE adds, and the file the pair writes back. The byte capacity is +# a RULE where a number would not travel, a record's byte cost is +# the port's own, so `full` and `short` say what a leg DOES rather +# than how many bytes it declares, and the id list, being a COUNT, +# travels as one. The message row is the form-2 read, whose save is +# the FILE form (§3.3). FORMAT.md states both shapes. +# # report # bytes read by a type that did not write them — the evolution class # (§4). The counters are the whole expectation, and they live in @@ -903,3 +914,53 @@ refusal block_count_past_extent bad_layout refusal block_count_past_maximum bad_layout refusal block_offset_overflow bad_layout refusal block_extent_padding truncated + +# ---- RETAIN-UNKNOWN (docs/SPEC-TABLES.md §6.6) ------------------------------ +# +# THE RT SET is the reader and the two newer builds: RT1 cannot name what RT2 +# and RT3 wrote, and every row below is one §6.6 owes. The wires are the same +# pinned vectors test/tables/retain_main.cpp writes, and the counters are the +# hand-authored expectation. +# +# `capacity` is `full`, a buffer the whole load fits in, or `short`, ONE BYTE +# SHORT of the last record: a record's byte cost is the port's own, so the +# short row is stated as a rule a leg applies rather than as a number two +# layouts would read differently. `ids` is `full` or the ENTRIES the caller's +# retained-id list holds, which is a COUNT and is comparable. +unit tblrt1 test/tables/RT1.schema +unit tblrt2 test/tables/RT2.schema +unit tblrt3 test/tables/RT3.schema + +# THE CONNECTION the message-form row resolves against: RT2's own announcement, +# read by RT1, which is a receiver meeting a LARGER unit (§3.3). +connection retain_conn tblrt2 0x59185ac86252d03a testdata/wire/tables/retain_conn.bin + +# THE ROUND TRIP AT DEPTH, and the two capacities. The first save is a pinned +# byte string of its own: moving a field changes the order ids are first used +# in, so the trailer is permuted and every reference moves with it. +retain retain_round_trip tblrt1 Node testdata/wire/tables/retain_rt2.bin full full 9,2,11 0 testdata/wire/tables/retain_rt1_save.bin +retain retain_truncated tblrt1 Node testdata/wire/tables/retain_rt2.bin short full 8,3,11 0 - +retain retain_ids_short tblrt1 Node testdata/wire/tables/retain_rt2.bin full 2 9,2,11 1 testdata/wire/tables/retain_rt1_save_id_short.bin + +# THE FIVE EXCLUDED CLASSES a wire can carry to the unknown arm, one row each +# and one retain_lost each, with `retained` unmoved. The NODE-INDEX class's row +# is the RECURSIVE shape, which is the widest case it has: an unknown outer +# table holding a nested pointer three bodies down. The SIXTH class, the +# reserved node-table field, has no row, because it is excluded by construction +# and a test cannot reach it. +retain retain_node_index tblrt1 Node testdata/wire/tables/retain_excluded_2.bin full full 0,1,1 0 - +retain retain_enum_variant tblrt1 Node testdata/wire/tables/retain_excluded_3.bin full full 0,1,1 0 - +retain retain_union_arm tblrt1 Node testdata/wire/tables/retain_excluded_4.bin full full 0,1,1 0 - +retain retain_keyed_slot tblrt1 Node testdata/wire/tables/retain_excluded_5.bin full full 0,1,1 0 - +retain retain_node_record tblrt1 Node testdata/wire/tables/retain_rt3.bin full full 0,1,1 0 - + +# THE MESSAGE FORM'S TAIL (§3.3): a form-2 batch loaded with retention against +# the connection's announced vocabulary and saved as FILES, which is the one +# direction retention crosses the forms. The batch's two bodies take one region +# and one retention buffer each, and the saves ride back to back in body order. +# `unknown` is SEVENTEEN where the two retention counters sum to sixteen, and +# the seventeenth is the plain read's own: a bit stream has to be walked past, +# so the DISCARDED SLOT's element body is decoded where a file's reader steps +# over it by its length, and the field inside it that RT1 cannot name counts +# there. +retain-message retain_message_tail retain_conn tblrt1 Node testdata/wire/tables/retain_message.bin full full 10,6,17 0 testdata/wire/tables/retain_message_save_0.bin,testdata/wire/tables/retain_message_save_1.bin diff --git a/testdata/golden/tables/arms/CarryTable.h b/testdata/golden/tables/arms/CarryTable.h index 57dcdce4f..1f23ac0c5 100644 --- a/testdata/golden/tables/arms/CarryTable.h +++ b/testdata/golden/tables/arms/CarryTable.h @@ -4241,6 +4241,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace armdemo #endif // ARMDEMO_SCHEMA_TABLE_RETAIN @@ -5367,18 +5799,22 @@ template inline int64_t LeafMeasureBodyRetain( const Ctx & ctx, c template inline bool LeafSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Leaf & value, TableRetain * retain, const TableRetainPath & path ); template inline bool LeafSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Leaf & value, TableRetain * retain, const TableRetainPath & path ); inline bool LeafLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Leaf & value, TableRetain * retain, const TableRetainPath & path ); +inline bool LeafLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Leaf & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t HolderMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Holder & value, TableRetain * retain, const TableRetainPath & path ); template inline bool HolderSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Holder & value, TableRetain * retain, const TableRetainPath & path ); template inline bool HolderSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Holder & value, TableRetain * retain, const TableRetainPath & path ); inline bool HolderLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Holder & value, TableRetain * retain, const TableRetainPath & path ); +inline bool HolderLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Holder & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t HandMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Hand & value, TableRetain * retain, const TableRetainPath & path ); template inline bool HandSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Hand & value, TableRetain * retain, const TableRetainPath & path ); template inline bool HandSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Hand & value, TableRetain * retain, const TableRetainPath & path ); inline bool HandLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Hand & value, TableRetain * retain, const TableRetainPath & path ); +inline bool HandLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Hand & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t ChainMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Chain & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ChainSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chain & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ChainSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chain & value, TableRetain * retain, const TableRetainPath & path ); inline bool ChainLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Chain & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ChainLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Chain & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t LeafMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Leaf & value ) @@ -12203,6 +12639,92 @@ inline bool LeafLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Lea } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool LeafLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Leaf & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + LeafReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3e7884bf4f412c6full: // items + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 4 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.items, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + int32_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + ( *slot ) = decoded_v_2; + } + } + TableListFillEnd( fill ); + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t HolderMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Holder & value, TableRetain * retain, const TableRetainPath & path ) { @@ -12530,13 +13052,192 @@ inline bool HolderLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, H } } -template -inline int64_t HandMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Hand & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool HolderLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Holder & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.entries_count < 0 || value.entries_count > 2 ) { return -1; } // storage invariant - if ( value.entries_count > 0 ) - { + (void) nodes; (void) index_bits; + HolderReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xafcd9a3b099f8ef0ull: // carry + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 15 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 0 ); + { + uint64_t arm_ref = 0; + if ( !r.get( arm_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm; + if ( arm_ref == 0 ) { value.carry.type = CarryType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref, arm ) ) { report->malformed = true; return false; } + else + { + switch ( arm.id ) + { + case 0x24ad84ada20208d5ull: // leaf + { + if ( arm.kind != 13 || arm.elem_kind != 0 ) + { + value.carry.type = CarryType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.carry.type = CarryType::Leaf; + if ( !LeafLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, value.carry.leaf, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm.kind != 4 || arm.elem_kind != 0 ) + { + if ( TableKindWidens( arm.kind, 4 ) ) + { + value.carry.type = CarryType::Plain; + memset( (void *) &value.carry.plain, 0, sizeof( value.carry.plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.carry.plain = decoded_v_2; + } + report->widened++; + break; + } + value.carry.type = CarryType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.carry.type = CarryType::Plain; + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.carry.plain = decoded_v_2; + } + break; + } + default: + value.carry.type = CarryType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + } + } + break; + } + case 0xd94c56ef0798d723ull: // tail + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 4 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.tail, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + int32_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + ( *slot ) = decoded_v_2; + } + } + TableListFillEnd( fill ); + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t HandMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Hand & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.entries_count < 0 || value.entries_count > 2 ) { return -1; } // storage invariant + if ( value.entries_count > 0 ) + { const uint64_t ref_entries = ids.ref( 0xc5b2a72c0845a253ull ); int64_t body_entries = 0; body_entries += 1 + TableLebBytes( (uint64_t) ( value.entries_count ) ); // the element kind byte and the count @@ -12851,50 +13552,255 @@ inline bool HandLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Han } } -template -inline int64_t ChainMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Chain & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool HandLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Hand & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body + (void) nodes; (void) index_bits; + HandReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - // links: a kind 14 array of kind 15 elements, INDEX order (§2.9) - TableListCursor cursor_links = TableListElements( ctx, value.links ); - if ( !cursor_links.ok ) { return -1; } // the slot and the head disagree - if ( cursor_links.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - const uint64_t ref_links = ids.ref( 0x5cc201a9f1b8274eull ); - int64_t body_links = 0; - body_links += 1 + TableLebBytes( (uint64_t) ( cursor_links.count ) ); // the element kind byte and the count - for ( int32_t elem_i_links = 0; elem_i_links < cursor_links.count; elem_i_links++ ) + case 0xc5b2a72c0845a253ull: // entries { - if ( cursor_links[elem_i_links].type == CarryType::None ) { body_links += 1; } // a None element is the zero reference in its place - else + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 15 ) { - switch ( cursor_links[elem_i_links].type ) + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) { - case CarryType::None: break; - case CarryType::Leaf: + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + Carry scratch; { - int64_t arm_payload_linksu = 0; - const uint64_t arm_ref_linksu = ids.ref( 0x24ad84ada20208d5ull ); + uint64_t arm_ref_2 = 0; + if ( !r.get( arm_ref_2, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm_2; + if ( arm_ref_2 == 0 ) { ( in_bounds ? value.entries[i] : scratch ).type = CarryType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref_2, arm_2 ) ) { report->malformed = true; return false; } + else { - const int64_t arm_body_linksu = LeafMeasureBodyRetain( ctx, numbering, ids, cursor_links[elem_i_links].leaf, retain, TableRetainStepInto( TableRetainStepInto( path, 0, (uint32_t) ( elem_i_links ) ), 0, (uint32_t) ( 0 ) ) ); - if ( arm_body_linksu < 0 ) { return -1; } - arm_payload_linksu += arm_body_linksu; // the arm's own table body (§3) - } - body_links += TableLebBytes( arm_ref_linksu ) + 1 + TableLebBytes( (uint64_t) ( arm_payload_linksu ) ) + ( arm_payload_linksu ); - break; - } - case CarryType::Plain: - { - int64_t arm_payload_linksu = 0; - const uint64_t arm_ref_linksu = ids.ref( 0xfd4d194e1652b207ull ); - arm_payload_linksu += 4; // int32 - body_links += TableLebBytes( arm_ref_linksu ) + 1 + TableLebBytes( (uint64_t) ( arm_payload_linksu ) ) + ( arm_payload_linksu ); - break; - } - default: return -1; // invalid tag — the write side refuses it too - } - } + switch ( arm_2.id ) + { + case 0x24ad84ada20208d5ull: // leaf + { + if ( arm_2.kind != 13 || arm_2.elem_kind != 0 ) + { + ( in_bounds ? value.entries[i] : scratch ).type = CarryType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( in_bounds ? value.entries[i] : scratch ).type = CarryType::Leaf; + if ( !LeafLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( in_bounds ? value.entries[i] : scratch ).leaf, ( in_bounds ? retain : NULL ), TableRetainStepInto( TableRetainStepInto( path, 0, (uint32_t) ( i ) ), 0, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm_2.kind != 4 || arm_2.elem_kind != 0 ) + { + if ( TableKindWidens( arm_2.kind, 4 ) ) + { + ( in_bounds ? value.entries[i] : scratch ).type = CarryType::Plain; + memset( (void *) &( in_bounds ? value.entries[i] : scratch ).plain, 0, sizeof( ( in_bounds ? value.entries[i] : scratch ).plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( in_bounds ? value.entries[i] : scratch ).plain = decoded_v_3; + } + report->widened++; + break; + } + ( in_bounds ? value.entries[i] : scratch ).type = CarryType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( in_bounds ? value.entries[i] : scratch ).type = CarryType::Plain; + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( in_bounds ? value.entries[i] : scratch ).plain = decoded_v_3; + } + break; + } + default: + ( in_bounds ? value.entries[i] : scratch ).type = CarryType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + } + } + } + value.entries_count = kept; + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t ChainMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Chain & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // links: a kind 14 array of kind 15 elements, INDEX order (§2.9) + TableListCursor cursor_links = TableListElements( ctx, value.links ); + if ( !cursor_links.ok ) { return -1; } // the slot and the head disagree + if ( cursor_links.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_links = ids.ref( 0x5cc201a9f1b8274eull ); + int64_t body_links = 0; + body_links += 1 + TableLebBytes( (uint64_t) ( cursor_links.count ) ); // the element kind byte and the count + for ( int32_t elem_i_links = 0; elem_i_links < cursor_links.count; elem_i_links++ ) + { + if ( cursor_links[elem_i_links].type == CarryType::None ) { body_links += 1; } // a None element is the zero reference in its place + else + { + switch ( cursor_links[elem_i_links].type ) + { + case CarryType::None: break; + case CarryType::Leaf: + { + int64_t arm_payload_linksu = 0; + const uint64_t arm_ref_linksu = ids.ref( 0x24ad84ada20208d5ull ); + { + const int64_t arm_body_linksu = LeafMeasureBodyRetain( ctx, numbering, ids, cursor_links[elem_i_links].leaf, retain, TableRetainStepInto( TableRetainStepInto( path, 0, (uint32_t) ( elem_i_links ) ), 0, (uint32_t) ( 0 ) ) ); + if ( arm_body_linksu < 0 ) { return -1; } + arm_payload_linksu += arm_body_linksu; // the arm's own table body (§3) + } + body_links += TableLebBytes( arm_ref_linksu ) + 1 + TableLebBytes( (uint64_t) ( arm_payload_linksu ) ) + ( arm_payload_linksu ); + break; + } + case CarryType::Plain: + { + int64_t arm_payload_linksu = 0; + const uint64_t arm_ref_linksu = ids.ref( 0xfd4d194e1652b207ull ); + arm_payload_linksu += 4; // int32 + body_links += TableLebBytes( arm_ref_linksu ) + 1 + TableLebBytes( (uint64_t) ( arm_payload_linksu ) ) + ( arm_payload_linksu ); + break; + } + default: return -1; // invalid tag — the write side refuses it too + } + } } bytes += TableLebBytes( ref_links ) + 1 + TableLebBytes( (uint64_t) ( body_links ) ) + ( body_links ); } @@ -13186,77 +14092,279 @@ inline bool ChainLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Ch } } -// LeafNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void LeafNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) -{ - // the node's own EXTENT, where its lists' and maps' arrays are carved - // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). - // The tool's path carries a worker instead: there the arrays are the - // arena's. - TableExtentCarve carve; - carve.worker = nodes.worker; - if ( carve.worker == NULL ) - { - TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse - const int64_t storage = LeafNodeStorage( type_id, r.size, reason ); - const int64_t record = storage > 0 ? LeafNodeRecordBytes( type_id ) : 0; - carve.at = at + record; - carve.left = storage > record ? storage - record : 0; - } - nodes.carve = &carve; - (void) nodes; // every node this root can name is a FIXED table - (void) retain; (void) node; - switch ( type_id ) - { - default: break; - } - nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done -} - -// LeafLoadRetain: decode the tolerant wire into the caller's exact-sized region and -// return the root. LOAD IS A SCAN, and that is the whole of its bound: it -// follows no reference, so there is no depth cap, no visited set and no -// ordering rule on the indices. Partial results are kept, as everywhere on -// this wire — the report says what happened. NULL means the CALLER's buffer -// was wrong. -// UNDER RETENTION it also fills the caller's two stores with the fields -// this build cannot name, and the report carries what it could not keep -// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the -// reader's data is exactly what it would have been with retention off. -inline const Leaf * LeafLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ChainLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Chain & value, TableRetain * retain, const TableRetainPath & path ) { - TableReport ignored; - TableReport * out = report != NULL ? report : &ignored; - // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: - // a file that is both a newer form and damaged is a REFUSAL and never - // damage (docs/SPEC-TABLES.md §3). - TableIdTable ids_table; - int64_t body_bytes = 0; - const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); - if ( verdict != TableOpenOk ) - { - if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } - return NULL; - } - if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + (void) nodes; (void) index_bits; + ChainReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - out->malformed = true; // a byte no field claims, before the table (§3) - return NULL; - } - const uint8_t * const wire = wire_file + 1; - const int64_t wire_bytes = body_bytes; - if ( region == NULL || region_bytes < (int64_t) sizeof( Leaf ) ) { out->malformed = true; return NULL; } - if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } - memset( region, 0, (size_t) region_bytes ); - uint64_t type_id = 0; - const uint8_t * body = NULL; - int64_t length = 0; - - // the record count and the data bytes, from the FRAMING alone + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x5cc201a9f1b8274eull: // links + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 15 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableListFill fill = TableListFillBegin( nodes, value.links, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Carry * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t arm_ref_2 = 0; + if ( !r.get( arm_ref_2, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm_2; + if ( arm_ref_2 == 0 ) { ( *slot ).type = CarryType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref_2, arm_2 ) ) { report->malformed = true; return false; } + else + { + switch ( arm_2.id ) + { + case 0x24ad84ada20208d5ull: // leaf + { + if ( arm_2.kind != 13 || arm_2.elem_kind != 0 ) + { + ( *slot ).type = CarryType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( *slot ).type = CarryType::Leaf; + if ( !LeafLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( *slot ).leaf, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm_2.kind != 4 || arm_2.elem_kind != 0 ) + { + if ( TableKindWidens( arm_2.kind, 4 ) ) + { + ( *slot ).type = CarryType::Plain; + memset( (void *) &( *slot ).plain, 0, sizeof( ( *slot ).plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( *slot ).plain = decoded_v_3; + } + report->widened++; + break; + } + ( *slot ).type = CarryType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( *slot ).type = CarryType::Plain; + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( *slot ).plain = decoded_v_3; + } + break; + } + default: + ( *slot ).type = CarryType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + } + } + } + TableListFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// LeafNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void LeafNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = LeafNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? LeafNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + (void) retain; (void) node; + switch ( type_id ) + { + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// LeafLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Leaf * LeafLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Leaf ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed int64_t root_extent = 0; if ( !LeafWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } @@ -13330,33 +14438,209 @@ inline const Leaf * LeafLoadRetain( uint8_t * region, int64_t region_bytes, cons if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } } - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + LeafNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Leaf ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + LeafLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// LeafNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool LeafNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + LeafNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !LeafNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// LeafLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool LeafLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Leaf * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !LeafMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = LeafNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + LeafNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !LeafMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Leaf ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xa5c085b4bb73d1b5ull; + Leaf * root = new ( region + used ) Leaf; // lifetime only: LoadMessageBody's first act is LeafReset + LeafReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Leaf ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - LeafNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !LeafNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Leaf ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - LeafLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return LeafLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// LeafLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool LeafLoadRetainMessages( const Leaf ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !LeafLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // LeafMeasureRetain and LeafSaveRetain: the pair, with the retained tail in @@ -13580,71 +14864,247 @@ inline const Holder * HolderLoadRetain( uint8_t * region, int64_t region_bytes, // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Holder ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = HolderNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + HolderNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + HolderNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Holder ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + HolderLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// HolderNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool HolderNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + HolderNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !HolderNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// HolderLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool HolderLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Holder * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !HolderMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = HolderNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + HolderNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !HolderMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Holder ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xa1f526c44ead452full; + Holder * root = new ( region + used ) Holder; // lifetime only: LoadMessageBody's first act is HolderReset + HolderReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Holder ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Holder ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = HolderNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - HolderNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - HolderNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !HolderNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Holder ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - HolderLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return HolderLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// HolderLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool HolderLoadRetainMessages( const Holder ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !HolderLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // HolderMeasureRetain and HolderSaveRetain: the pair, with the retained tail in @@ -13868,71 +15328,247 @@ inline const Hand * HandLoadRetain( uint8_t * region, int64_t region_bytes, cons // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Hand ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = HandNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + HandNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + HandNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Hand ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + HandLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// HandNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool HandNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + HandNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !HandNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// HandLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool HandLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Hand * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !HandMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = HandNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + HandNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !HandMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Hand ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x58ba95d8757c67c0ull; + Hand * root = new ( region + used ) Hand; // lifetime only: LoadMessageBody's first act is HandReset + HandReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Hand ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Hand ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = HandNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - HandNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - HandNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !HandNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Hand ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - HandLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return HandLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// HandLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool HandLoadRetainMessages( const Hand ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !HandLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // HandMeasureRetain and HandSaveRetain: the pair, with the retained tail in @@ -14223,6 +15859,182 @@ inline const Chain * ChainLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// ChainNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool ChainNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + ChainNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !ChainNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// ChainLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool ChainLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Chain * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !ChainMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = ChainNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + ChainNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !ChainMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Chain ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x670377dcbe2f82b6ull; + Chain * root = new ( region + used ) Chain; // lifetime only: LoadMessageBody's first act is ChainReset + ChainReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Chain ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !ChainNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return ChainLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// ChainLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool ChainLoadRetainMessages( const Chain ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !ChainLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // ChainMeasureRetain and ChainSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/arms/GateTable.h b/testdata/golden/tables/arms/GateTable.h index 6e04cdf93..0a2f7f701 100644 --- a/testdata/golden/tables/arms/GateTable.h +++ b/testdata/golden/tables/arms/GateTable.h @@ -4241,6 +4241,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace armdemo #endif // ARMDEMO_SCHEMA_TABLE_RETAIN @@ -5328,10 +5760,12 @@ template inline bool TableNodeMessageSave( const Ctx & ctx, const inline int64_t OnlyMeasureBodyRetain( TableRetainIds & ids, const Only & value, TableRetain * retain, const TableRetainPath & path ); ARMDEMO_TABLE_INLINE bool OnlySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Only & value, TableRetain * retain, const TableRetainPath & path ); ARMDEMO_TABLE_INLINE bool OnlyLoadBodyRetain( TableReader & r, Only & value, TableRetain * retain, const TableRetainPath & path ); +inline bool OnlyLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Only & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t GateMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Gate & value, TableRetain * retain, const TableRetainPath & path ); template inline bool GateSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Gate & value, TableRetain * retain, const TableRetainPath & path ); template inline bool GateSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Gate & value, TableRetain * retain, const TableRetainPath & path ); inline bool GateLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Gate & value, TableRetain * retain, const TableRetainPath & path ); +inline bool GateLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Gate & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t OnlyMeasureBody( TableIds & ids, const Only & value ) { @@ -7609,6 +8043,98 @@ ARMDEMO_TABLE_INLINE bool OnlyLoadBodyRetain( TableReader & r, Only & value, Tab } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool OnlyLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Only & value, TableRetain * retain, const TableRetainPath & path ) +{ + OnlyReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xaf63ea4c86020456ull: // w + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.w = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.w = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t GateMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Gate & value, TableRetain * retain, const TableRetainPath & path ) { @@ -7914,11 +8440,217 @@ inline bool GateLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Gat } } -// GateNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool GateLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Gate & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + GateReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x891da1f305deb858ull: // reach + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 15 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 0 ); + { + uint64_t arm_ref = 0; + if ( !r.get( arm_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm; + if ( arm_ref == 0 ) { value.reach.type = ReachType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref, arm ) ) { report->malformed = true; return false; } + else + { + switch ( arm.id ) + { + case 0x072ddab46af04ab7ull: // only + { + if ( arm.kind != 17 || arm.elem_kind != 0 ) + { + value.reach.type = ReachType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.reach.type = ReachType::Only; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.reach.only, node_index_2, 0x31badbc06c5f0b97ull, report ); // *Only + } + break; + } + case 0xfa04f4ef1995407eull: // text + { + if ( arm.kind != 17 || arm.elem_kind != 0 ) + { + value.reach.type = ReachType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.reach.type = ReachType::Text; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.reach.text, node_index_2, kTableStringTypeId, report ); // *string + } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm.kind != 4 || arm.elem_kind != 0 ) + { + if ( TableKindWidens( arm.kind, 4 ) ) + { + value.reach.type = ReachType::Plain; + memset( (void *) &value.reach.plain, 0, sizeof( value.reach.plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.reach.plain = decoded_v_2; + } + report->widened++; + break; + } + value.reach.type = ReachType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.reach.type = ReachType::Plain; + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.reach.plain = decoded_v_2; + } + break; + } + default: + value.reach.type = ReachType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + } + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// GateNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. inline void GateNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) { // the node's own EXTENT, where its lists' and maps' arrays are carved @@ -8096,6 +8828,179 @@ inline const Gate * GateLoadRetain( uint8_t * region, int64_t region_bytes, cons return root; } +// GateNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool GateNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + GateNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !GateNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x31badbc06c5f0b97ull: ok = OnlyLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Only *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Only + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// GateLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool GateLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Gate * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !GateMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = GateNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + GateNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Gate ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xb5877a7e05bbfed2ull; + Gate * root = new ( region + used ) Gate; // lifetime only: LoadMessageBody's first act is GateReset + GateReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Gate ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !GateNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return GateLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// GateLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool GateLoadRetainMessages( const Gate ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !GateLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // GateMeasureRetain and GateSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/arms/NestTable.h b/testdata/golden/tables/arms/NestTable.h index c65182742..83e7a3a7b 100644 --- a/testdata/golden/tables/arms/NestTable.h +++ b/testdata/golden/tables/arms/NestTable.h @@ -4242,6 +4242,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace armdemo #endif // ARMDEMO_SCHEMA_TABLE_RETAIN @@ -5344,10 +5776,12 @@ template inline int64_t TwigMeasureBodyRetain( const Ctx & ctx, c template inline bool TwigSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Twig & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TwigSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Twig & value, TableRetain * retain, const TableRetainPath & path ); inline bool TwigLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Twig & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TwigLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Twig & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t NestMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Nest & value, TableRetain * retain, const TableRetainPath & path ); template inline bool NestSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Nest & value, TableRetain * retain, const TableRetainPath & path ); template inline bool NestSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Nest & value, TableRetain * retain, const TableRetainPath & path ); inline bool NestLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Nest & value, TableRetain * retain, const TableRetainPath & path ); +inline bool NestLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Nest & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t TwigMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Twig & value ) @@ -9124,54 +9558,190 @@ inline bool TwigLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Twi } } -template -inline int64_t NestMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Nest & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TwigLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Twig & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.outer.type != OuterType::None ) // None elides — the absence of the field is the None + (void) nodes; (void) index_bits; + TwigReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - bytes += TableLebBytes( ids.ref( 0x30e6ed678f5e1bd4ull ) ) + 1; - switch ( value.outer.type ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - case OuterType::None: break; - case OuterType::Inner: + case 0x2d4c068f5f889287ull: // inner { - int64_t arm_payload = 0; - const uint64_t arm_ref = ids.ref( 0x2d4c068f5f889287ull ); - if ( value.outer.inner.type == InnerType::None ) { arm_payload += 1; } // the inner union's None: L = 1 and that one zero byte (§3) - else + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 15 || entry.elem_kind != 0 ) { - switch ( value.outer.inner.type ) + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 0 ); + { + uint64_t arm_ref = 0; + if ( !r.get( arm_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm; + if ( arm_ref == 0 ) { value.inner.type = InnerType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref, arm ) ) { report->malformed = true; return false; } + else { - case InnerType::None: break; - case InnerType::Leaf: + switch ( arm.id ) { - int64_t arm_payloada = 0; - const uint64_t arm_refa = ids.ref( 0x24ad84ada20208d5ull ); + case 0x24ad84ada20208d5ull: // leaf { - const int64_t arm_bodya = LeafMeasureBodyRetain( ctx, numbering, ids, value.outer.inner.leaf, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ); - if ( arm_bodya < 0 ) { return -1; } - arm_payloada += arm_bodya; // the arm's own table body (§3) + if ( arm.kind != 13 || arm.elem_kind != 0 ) + { + value.inner.type = InnerType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.inner.type = InnerType::Leaf; + if ( !LeafLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, value.inner.leaf, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ) ) { return false; } + break; } - arm_payload += TableLebBytes( arm_refa ) + 1 + TableLebBytes( (uint64_t) ( arm_payloada ) ) + ( arm_payloada ); - break; - } - case InnerType::Plain: - { - int64_t arm_payloada = 0; - const uint64_t arm_refa = ids.ref( 0xfd4d194e1652b207ull ); - arm_payloada += 4; // int32 - arm_payload += TableLebBytes( arm_refa ) + 1 + TableLebBytes( (uint64_t) ( arm_payloada ) ) + ( arm_payloada ); - break; - } - default: return -1; // invalid tag — the write side refuses it too - } - } - bytes += TableLebBytes( arm_ref ) + 1 + TableLebBytes( (uint64_t) ( arm_payload ) ) + ( arm_payload ); - break; - } - case OuterType::Plain: - { + case 0xfd4d194e1652b207ull: // plain + { + if ( arm.kind != 4 || arm.elem_kind != 0 ) + { + if ( TableKindWidens( arm.kind, 4 ) ) + { + value.inner.type = InnerType::Plain; + memset( (void *) &value.inner.plain, 0, sizeof( value.inner.plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.inner.plain = decoded_v_2; + } + report->widened++; + break; + } + value.inner.type = InnerType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.inner.type = InnerType::Plain; + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.inner.plain = decoded_v_2; + } + break; + } + default: + value.inner.type = InnerType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + } + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t NestMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Nest & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.outer.type != OuterType::None ) // None elides — the absence of the field is the None + { + bytes += TableLebBytes( ids.ref( 0x30e6ed678f5e1bd4ull ) ) + 1; + switch ( value.outer.type ) + { + case OuterType::None: break; + case OuterType::Inner: + { + int64_t arm_payload = 0; + const uint64_t arm_ref = ids.ref( 0x2d4c068f5f889287ull ); + if ( value.outer.inner.type == InnerType::None ) { arm_payload += 1; } // the inner union's None: L = 1 and that one zero byte (§3) + else + { + switch ( value.outer.inner.type ) + { + case InnerType::None: break; + case InnerType::Leaf: + { + int64_t arm_payloada = 0; + const uint64_t arm_refa = ids.ref( 0x24ad84ada20208d5ull ); + { + const int64_t arm_bodya = LeafMeasureBodyRetain( ctx, numbering, ids, value.outer.inner.leaf, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ); + if ( arm_bodya < 0 ) { return -1; } + arm_payloada += arm_bodya; // the arm's own table body (§3) + } + arm_payload += TableLebBytes( arm_refa ) + 1 + TableLebBytes( (uint64_t) ( arm_payloada ) ) + ( arm_payloada ); + break; + } + case InnerType::Plain: + { + int64_t arm_payloada = 0; + const uint64_t arm_refa = ids.ref( 0xfd4d194e1652b207ull ); + arm_payloada += 4; // int32 + arm_payload += TableLebBytes( arm_refa ) + 1 + TableLebBytes( (uint64_t) ( arm_payloada ) ) + ( arm_payloada ); + break; + } + default: return -1; // invalid tag — the write side refuses it too + } + } + bytes += TableLebBytes( arm_ref ) + 1 + TableLebBytes( (uint64_t) ( arm_payload ) ) + ( arm_payload ); + break; + } + case OuterType::Plain: + { int64_t arm_payload = 0; const uint64_t arm_ref = ids.ref( 0xfd4d194e1652b207ull ); arm_payload += 4; // int32 @@ -9584,129 +10154,386 @@ inline bool NestLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Nes } } -// TwigNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void TwigNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) -{ - // the node's own EXTENT, where its lists' and maps' arrays are carved - // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). - // The tool's path carries a worker instead: there the arrays are the - // arena's. - TableExtentCarve carve; - carve.worker = nodes.worker; - if ( carve.worker == NULL ) - { - TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse - const int64_t storage = TwigNodeStorage( type_id, r.size, reason ); - const int64_t record = storage > 0 ? TwigNodeRecordBytes( type_id ) : 0; - carve.at = at + record; - carve.left = storage > record ? storage - record : 0; - } - nodes.carve = &carve; - (void) nodes; // every node this root can name is a FIXED table - (void) retain; (void) node; - switch ( type_id ) - { - default: break; - } - nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done -} - -// TwigLoadRetain: decode the tolerant wire into the caller's exact-sized region and -// return the root. LOAD IS A SCAN, and that is the whole of its bound: it -// follows no reference, so there is no depth cap, no visited set and no -// ordering rule on the indices. Partial results are kept, as everywhere on -// this wire — the report says what happened. NULL means the CALLER's buffer -// was wrong. -// UNDER RETENTION it also fills the caller's two stores with the fields -// this build cannot name, and the report carries what it could not keep -// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the -// reader's data is exactly what it would have been with retention off. -inline const Twig * TwigLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool NestLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Nest & value, TableRetain * retain, const TableRetainPath & path ) { - TableReport ignored; - TableReport * out = report != NULL ? report : &ignored; - // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: - // a file that is both a newer form and damaged is a REFUSAL and never - // damage (docs/SPEC-TABLES.md §3). - TableIdTable ids_table; - int64_t body_bytes = 0; - const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); - if ( verdict != TableOpenOk ) - { - if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } - return NULL; - } - if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) - { - out->malformed = true; // a byte no field claims, before the table (§3) - return NULL; - } - const uint8_t * const wire = wire_file + 1; - const int64_t wire_bytes = body_bytes; - if ( region == NULL || region_bytes < (int64_t) sizeof( Twig ) ) { out->malformed = true; return NULL; } - if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } - memset( region, 0, (size_t) region_bytes ); - uint64_t type_id = 0; - const uint8_t * body = NULL; - int64_t length = 0; - - // the record count and the data bytes, from the FRAMING alone - TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed - int64_t root_extent = 0; - if ( !TwigWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } - int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Twig ) ) + root_extent ); - int64_t records = 0; - { - TableReport counting; - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); - while ( TableNodeScanNext( scan, type_id, body, length ) ) - { - records++; - int64_t storage = TwigNodeStorage( type_id, length, reason ); - if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } - if ( storage > 0 ) { data += storage; } - } - } - int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); - if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } - - TableNodeMap nodes; - nodes.base = region; - nodes.entries = (const TableNodeDirEntry *) ( region + data ); - nodes.count = records + 1; - TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); - directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) - directory[0].type_id = 0x9f8e22fb612a2f78ull; - Twig * root = new ( region ) Twig; // lifetime only: LoadBody's first act is TwigReset - TwigReset( *root ); - - // LoadRetain RESETS BOTH STORES and writes into neither id list: a - // retained record carries its field's identity in the record itself, - // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer - // belongs to this region from here on. - TableRetainReset( retain, nodes, region ); - - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + (void) nodes; (void) index_bits; + NestReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Twig ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - int64_t storage = TwigNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) + case 0x30e6ed678f5e1bd4ull: // outer { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 15 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 0 ); + { + uint64_t arm_ref = 0; + if ( !r.get( arm_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm; + if ( arm_ref == 0 ) { value.outer.type = OuterType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref, arm ) ) { report->malformed = true; return false; } + else + { + switch ( arm.id ) + { + case 0x2d4c068f5f889287ull: // inner + { + if ( arm.kind != 15 || arm.elem_kind != 0 ) + { + value.outer.type = OuterType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.outer.type = OuterType::Inner; + { + uint64_t arm_ref_2 = 0; + if ( !r.get( arm_ref_2, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm_2; + if ( arm_ref_2 == 0 ) { value.outer.inner.type = InnerType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref_2, arm_2 ) ) { report->malformed = true; return false; } + else + { + switch ( arm_2.id ) + { + case 0x24ad84ada20208d5ull: // leaf + { + if ( arm_2.kind != 13 || arm_2.elem_kind != 0 ) + { + value.outer.inner.type = InnerType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + value.outer.inner.type = InnerType::Leaf; + if ( !LeafLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, value.outer.inner.leaf, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm_2.kind != 4 || arm_2.elem_kind != 0 ) + { + if ( TableKindWidens( arm_2.kind, 4 ) ) + { + value.outer.inner.type = InnerType::Plain; + memset( (void *) &value.outer.inner.plain, 0, sizeof( value.outer.inner.plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + value.outer.inner.plain = decoded_v_3; + } + report->widened++; + break; + } + value.outer.inner.type = InnerType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + value.outer.inner.type = InnerType::Plain; + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + value.outer.inner.plain = decoded_v_3; + } + break; + } + default: + value.outer.inner.type = InnerType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + } + } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm.kind != 4 || arm.elem_kind != 0 ) + { + if ( TableKindWidens( arm.kind, 4 ) ) + { + value.outer.type = OuterType::Plain; + memset( (void *) &value.outer.plain, 0, sizeof( value.outer.plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.outer.plain = decoded_v_2; + } + report->widened++; + break; + } + value.outer.type = OuterType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.outer.type = OuterType::Plain; + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.outer.plain = decoded_v_2; + } + break; + } + default: + value.outer.type = OuterType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + } + } + break; + } + case 0xd94c56ef0798d723ull: // tail + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 4 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.tail, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + int32_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + ( *slot ) = decoded_v_2; + } + } + TableListFillEnd( fill ); + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// TwigNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void TwigNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = TwigNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? TwigNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + (void) retain; (void) node; + switch ( type_id ) + { + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// TwigLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Twig * TwigLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Twig ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !TwigWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Twig ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = TwigNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0x9f8e22fb612a2f78ull; + Twig * root = new ( region ) Twig; // lifetime only: LoadBody's first act is TwigReset + TwigReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Twig ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = TwigNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; directory[k + 1].type_id = type_id; } else @@ -9718,43 +10545,219 @@ inline const Twig * TwigLoadRetain( uint8_t * region, int64_t region_bytes, cons } k++; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + TwigNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Twig ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + TwigLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// TwigNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool TwigNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + TwigNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !TwigNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// TwigLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool TwigLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Twig * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TwigMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = TwigNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + TwigNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !TwigMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Twig ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x9f8e22fb612a2f78ull; + Twig * root = new ( region + used ) Twig; // lifetime only: LoadMessageBody's first act is TwigReset + TwigReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Twig ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - TwigNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !TwigNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Twig ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - TwigLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return TwigLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// TwigLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool TwigLoadRetainMessages( const Twig ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !TwigLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // TwigMeasureRetain and TwigSaveRetain: the pair, with the retained tail in @@ -10045,6 +11048,182 @@ inline const Nest * NestLoadRetain( uint8_t * region, int64_t region_bytes, cons return root; } +// NestNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool NestNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + NestNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !NestNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// NestLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool NestLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Nest * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !NestMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = NestNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + NestNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !NestMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Nest ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x0fe46fc6a1b583abull; + Nest * root = new ( region + used ) Nest; // lifetime only: LoadMessageBody's first act is NestReset + NestReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Nest ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !NestNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return NestLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// NestLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool NestLoadRetainMessages( const Nest ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !NestLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // NestMeasureRetain and NestSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/arms/RingTable.h b/testdata/golden/tables/arms/RingTable.h index 9c838cc50..8b5467daf 100644 --- a/testdata/golden/tables/arms/RingTable.h +++ b/testdata/golden/tables/arms/RingTable.h @@ -4241,6 +4241,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace armdemo #endif // ARMDEMO_SCHEMA_TABLE_RETAIN @@ -5435,18 +5867,22 @@ template inline bool TableNodeMessageSave( const Ctx & ctx, const inline int64_t NodeMeasureBodyRetain( TableRetainIds & ids, const Node & value, TableRetain * retain, const TableRetainPath & path ); ARMDEMO_TABLE_INLINE bool NodeSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Node & value, TableRetain * retain, const TableRetainPath & path ); ARMDEMO_TABLE_INLINE bool NodeLoadBodyRetain( TableReader & r, Node & value, TableRetain * retain, const TableRetainPath & path ); +inline bool NodeLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Node & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t RingMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Ring & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RingSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Ring & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RingSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Ring & value, TableRetain * retain, const TableRetainPath & path ); inline bool RingLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Ring & value, TableRetain * retain, const TableRetainPath & path ); +inline bool RingLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Ring & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t RackMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Rack & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RackSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Rack & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RackSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Rack & value, TableRetain * retain, const TableRetainPath & path ); inline bool RackLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Rack & value, TableRetain * retain, const TableRetainPath & path ); +inline bool RackLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Rack & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t TrayMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Tray & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TraySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Tray & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TraySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Tray & value, TableRetain * retain, const TableRetainPath & path ); inline bool TrayLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Tray & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TrayLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Tray & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t NodeMeasureBody( TableIds & ids, const Node & value ) { @@ -11506,6 +11942,98 @@ ARMDEMO_TABLE_INLINE bool NodeLoadBodyRetain( TableReader & r, Node & value, Tab } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool NodeLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Node & value, TableRetain * retain, const TableRetainPath & path ) +{ + NodeReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xaf63eb4c86020609ull: // v + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.v = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.v = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t RingMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Ring & value, TableRetain * retain, const TableRetainPath & path ) { @@ -11853,11 +12381,217 @@ inline bool RingLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Rin } } -template -inline int64_t RackMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Rack & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool RingLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Ring & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + RingReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3e7884bf4f412c6full: // items + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 15 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableListFill fill = TableListFillBegin( nodes, value.items, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Slot * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t arm_ref_2 = 0; + if ( !r.get( arm_ref_2, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm_2; + if ( arm_ref_2 == 0 ) { ( *slot ).type = SlotType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref_2, arm_2 ) ) { report->malformed = true; return false; } + else + { + switch ( arm_2.id ) + { + case 0x3c2f1bbad18642adull: // node + { + if ( arm_2.kind != 17 || arm_2.elem_kind != 0 ) + { + ( *slot ).type = SlotType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( *slot ).type = SlotType::Node; + { + uint64_t node_index_3 = 0; + if ( !r.get( node_index_3, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( *slot ).node, node_index_3, 0x66bd1cc6d2f6b68dull, report ); // *Node + } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm_2.kind != 4 || arm_2.elem_kind != 0 ) + { + if ( TableKindWidens( arm_2.kind, 4 ) ) + { + ( *slot ).type = SlotType::Plain; + memset( (void *) &( *slot ).plain, 0, sizeof( ( *slot ).plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( *slot ).plain = decoded_v_3; + } + report->widened++; + break; + } + ( *slot ).type = SlotType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( *slot ).type = SlotType::Plain; + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( *slot ).plain = decoded_v_3; + } + break; + } + default: + ( *slot ).type = SlotType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + } + } + } + TableListFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t RackMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Rack & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { // items: a kind 14 array of kind 15 elements, INDEX order (§2.9) TableListCursor cursor_items = TableListElements( ctx, value.items ); if ( !cursor_items.ok ) { return -1; } // the slot and the head disagree @@ -12238,28 +12972,254 @@ inline bool RackLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Rac } } -template -inline int64_t TrayMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Tray & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool RackLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Rack & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.entries_count < 0 || value.entries_count > 2 ) { return -1; } // storage invariant - if ( value.entries_count > 0 ) + (void) nodes; (void) index_bits; + RackReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - const uint64_t ref_entries = ids.ref( 0xc5b2a72c0845a253ull ); - int64_t body_entries = 0; - body_entries += 1 + TableLebBytes( (uint64_t) ( value.entries_count ) ); // the element kind byte and the count - for ( int32_t elem_i = 0; elem_i < value.entries_count; elem_i++ ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - if ( value.entries[elem_i].type == SlotsType::None ) { body_entries += 1; } // a None element is the zero reference in its place - else + case 0x3e7884bf4f412c6full: // items { - switch ( value.entries[elem_i].type ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 15 ) { - case SlotsType::None: break; - case SlotsType::Many: + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableListFill fill = TableListFillBegin( nodes, value.items, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) { - int64_t arm_payloadu = 0; - const uint64_t arm_refu = ids.ref( 0x1f6459a2cea1fc02ull ); + Slots * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t arm_ref_2 = 0; + if ( !r.get( arm_ref_2, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm_2; + if ( arm_ref_2 == 0 ) { ( *slot ).type = SlotsType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref_2, arm_2 ) ) { report->malformed = true; return false; } + else + { + switch ( arm_2.id ) + { + case 0x1f6459a2cea1fc02ull: // many + { + if ( arm_2.kind != 14 || arm_2.elem_kind != 17 ) + { + ( *slot ).type = SlotsType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( *slot ).type = SlotsType::Many; + { + uint64_t n_3 = (uint64_t) arm_2.min; + const int64_t count_bits_3 = TableBitsRequired( arm_2.min, arm_2.max ); + if ( count_bits_3 > 0 ) + { + uint64_t raw_3 = 0; + if ( !r.get( raw_3, count_bits_3 ) ) { report->malformed = true; return false; } + n_3 = raw_3 + (uint64_t) arm_2.min; + } + if ( arm_2.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept_3 = 0; + if ( n_3 > (uint64_t) 2 ) { kept_3 = 2; report->clamped++; } else { kept_3 = (int32_t) n_3; } + const uint64_t walk_3 = n_3; + for ( uint64_t i_3 = 0; i_3 < walk_3; i_3++ ) + { + const bool in_bounds_3 = (int32_t) i_3 < kept_3; + TableRef scratch_3; + { + uint64_t node_index_4 = 0; + if ( !r.get( node_index_4, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds_3 ? ( *slot ).many.value[i_3] : scratch_3 ), node_index_4, 0x66bd1cc6d2f6b68dull, report ); // *Node + } + } + ( *slot ).many.value_count = kept_3; + } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm_2.kind != 4 || arm_2.elem_kind != 0 ) + { + if ( TableKindWidens( arm_2.kind, 4 ) ) + { + ( *slot ).type = SlotsType::Plain; + memset( (void *) &( *slot ).plain, 0, sizeof( ( *slot ).plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( *slot ).plain = decoded_v_3; + } + report->widened++; + break; + } + ( *slot ).type = SlotsType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( *slot ).type = SlotsType::Plain; + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( *slot ).plain = decoded_v_3; + } + break; + } + default: + ( *slot ).type = SlotsType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + } + } + } + TableListFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t TrayMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Tray & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.entries_count < 0 || value.entries_count > 2 ) { return -1; } // storage invariant + if ( value.entries_count > 0 ) + { + const uint64_t ref_entries = ids.ref( 0xc5b2a72c0845a253ull ); + int64_t body_entries = 0; + body_entries += 1 + TableLebBytes( (uint64_t) ( value.entries_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.entries_count; elem_i++ ) + { + if ( value.entries[elem_i].type == SlotsType::None ) { body_entries += 1; } // a None element is the zero reference in its place + else + { + switch ( value.entries[elem_i].type ) + { + case SlotsType::None: break; + case SlotsType::Many: + { + int64_t arm_payloadu = 0; + const uint64_t arm_refu = ids.ref( 0x1f6459a2cea1fc02ull ); if ( value.entries[elem_i].many.value_count < 0 || value.entries[elem_i].many.value_count > 2 ) { return -1; } // storage invariant arm_payloadu += 1 + TableLebBytes( (uint64_t) ( value.entries[elem_i].many.value_count ) ); // the element kind byte and the count for ( int32_t elem_iue = 0; elem_iue < value.entries[elem_i].many.value_count; elem_iue++ ) @@ -12609,63 +13569,292 @@ inline bool TrayLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Tra } } -// RingNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void RingNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) -{ - // the node's own EXTENT, where its lists' and maps' arrays are carved - // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). - // The tool's path carries a worker instead: there the arrays are the - // arena's. - TableExtentCarve carve; - carve.worker = nodes.worker; - if ( carve.worker == NULL ) - { - TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse - const int64_t storage = RingNodeStorage( type_id, r.size, reason ); - const int64_t record = storage > 0 ? RingNodeRecordBytes( type_id ) : 0; - carve.at = at + record; - carve.left = storage > record ? storage - record : 0; - } - nodes.carve = &carve; - (void) nodes; // every node this root can name is a FIXED table - switch ( type_id ) - { - case 0x66bd1cc6d2f6b68dull: NodeLoadBodyRetain( r, *(Node *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Node - default: break; - } - nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done -} - -// RingLoadRetain: decode the tolerant wire into the caller's exact-sized region and -// return the root. LOAD IS A SCAN, and that is the whole of its bound: it -// follows no reference, so there is no depth cap, no visited set and no -// ordering rule on the indices. Partial results are kept, as everywhere on -// this wire — the report says what happened. NULL means the CALLER's buffer -// was wrong. -// UNDER RETENTION it also fills the caller's two stores with the fields -// this build cannot name, and the report carries what it could not keep -// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the -// reader's data is exactly what it would have been with retention off. -inline const Ring * RingLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TrayLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Tray & value, TableRetain * retain, const TableRetainPath & path ) { - TableReport ignored; - TableReport * out = report != NULL ? report : &ignored; - // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: - // a file that is both a newer form and damaged is a REFUSAL and never - // damage (docs/SPEC-TABLES.md §3). - TableIdTable ids_table; - int64_t body_bytes = 0; - const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); - if ( verdict != TableOpenOk ) - { - if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } - return NULL; - } - if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + (void) nodes; (void) index_bits; + TrayReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xc5b2a72c0845a253ull: // entries + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 15 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + Slots scratch; + { + uint64_t arm_ref_2 = 0; + if ( !r.get( arm_ref_2, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm_2; + if ( arm_ref_2 == 0 ) { ( in_bounds ? value.entries[i] : scratch ).type = SlotsType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref_2, arm_2 ) ) { report->malformed = true; return false; } + else + { + switch ( arm_2.id ) + { + case 0x1f6459a2cea1fc02ull: // many + { + if ( arm_2.kind != 14 || arm_2.elem_kind != 17 ) + { + ( in_bounds ? value.entries[i] : scratch ).type = SlotsType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( in_bounds ? value.entries[i] : scratch ).type = SlotsType::Many; + { + uint64_t n_3 = (uint64_t) arm_2.min; + const int64_t count_bits_3 = TableBitsRequired( arm_2.min, arm_2.max ); + if ( count_bits_3 > 0 ) + { + uint64_t raw_3 = 0; + if ( !r.get( raw_3, count_bits_3 ) ) { report->malformed = true; return false; } + n_3 = raw_3 + (uint64_t) arm_2.min; + } + if ( arm_2.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept_3 = 0; + if ( n_3 > (uint64_t) 2 ) { kept_3 = 2; report->clamped++; } else { kept_3 = (int32_t) n_3; } + const uint64_t walk_3 = n_3; + for ( uint64_t i_3 = 0; i_3 < walk_3; i_3++ ) + { + const bool in_bounds_3 = (int32_t) i_3 < kept_3; + TableRef scratch_3; + { + uint64_t node_index_4 = 0; + if ( !r.get( node_index_4, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds_3 ? ( in_bounds ? value.entries[i] : scratch ).many.value[i_3] : scratch_3 ), node_index_4, 0x66bd1cc6d2f6b68dull, report ); // *Node + } + } + ( in_bounds ? value.entries[i] : scratch ).many.value_count = kept_3; + } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm_2.kind != 4 || arm_2.elem_kind != 0 ) + { + if ( TableKindWidens( arm_2.kind, 4 ) ) + { + ( in_bounds ? value.entries[i] : scratch ).type = SlotsType::Plain; + memset( (void *) &( in_bounds ? value.entries[i] : scratch ).plain, 0, sizeof( ( in_bounds ? value.entries[i] : scratch ).plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( in_bounds ? value.entries[i] : scratch ).plain = decoded_v_3; + } + report->widened++; + break; + } + ( in_bounds ? value.entries[i] : scratch ).type = SlotsType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( in_bounds ? value.entries[i] : scratch ).type = SlotsType::Plain; + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( in_bounds ? value.entries[i] : scratch ).plain = decoded_v_3; + } + break; + } + default: + ( in_bounds ? value.entries[i] : scratch ).type = SlotsType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + } + } + } + value.entries_count = kept; + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// RingNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void RingNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = RingNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? RingNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x66bd1cc6d2f6b68dull: NodeLoadBodyRetain( r, *(Node *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Node + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// RingLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Ring * RingLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) { out->malformed = true; // a byte no field claims, before the table (§3) return NULL; @@ -12715,71 +13904,247 @@ inline const Ring * RingLoadRetain( uint8_t * region, int64_t region_bytes, cons // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Ring ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = RingNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + RingNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + RingNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Ring ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + RingLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// RingNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool RingNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + RingNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !RingNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x66bd1cc6d2f6b68dull: ok = NodeLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Node *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Node + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// RingLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool RingLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Ring * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !RingMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = RingNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + RingNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !RingMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Ring ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xdcea4d2bfa11ceebull; + Ring * root = new ( region + used ) Ring; // lifetime only: LoadMessageBody's first act is RingReset + RingReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Ring ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Ring ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = RingNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - RingNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - RingNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !RingNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Ring ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - RingLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return RingLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// RingLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool RingLoadRetainMessages( const Ring ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !RingLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // RingMeasureRetain and RingSaveRetain: the pair, with the retained tail in @@ -13006,71 +14371,247 @@ inline const Rack * RackLoadRetain( uint8_t * region, int64_t region_bytes, cons // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Rack ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = RackNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + RackNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + RackNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Rack ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + RackLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// RackNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool RackNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + RackNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !RackNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x66bd1cc6d2f6b68dull: ok = NodeLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Node *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Node + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// RackLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool RackLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Rack * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !RackMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = RackNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + RackNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !RackMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Rack ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x986d012bd380b0b2ull; + Rack * root = new ( region + used ) Rack; // lifetime only: LoadMessageBody's first act is RackReset + RackReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Rack ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Rack ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = RackNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - RackNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - RackNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !RackNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Rack ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - RackLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return RackLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// RackLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool RackLoadRetainMessages( const Rack ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !RackLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // RackMeasureRetain and RackSaveRetain: the pair, with the retained tail in @@ -13364,6 +14905,178 @@ inline const Tray * TrayLoadRetain( uint8_t * region, int64_t region_bytes, cons return root; } +// TrayNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool TrayNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + TrayNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !TrayNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x66bd1cc6d2f6b68dull: ok = NodeLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Node *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Node + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// TrayLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool TrayLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Tray * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TrayMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = TrayNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + TrayNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Tray ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x85a6b5fb5284964dull; + Tray * root = new ( region + used ) Tray; // lifetime only: LoadMessageBody's first act is TrayReset + TrayReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Tray ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !TrayNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return TrayLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// TrayLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool TrayLoadRetainMessages( const Tray ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !TrayLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // TrayMeasureRetain and TraySaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/blobs/AssetsTable.h b/testdata/golden/tables/blobs/AssetsTable.h index 47216b624..0fe943c5e 100644 --- a/testdata/golden/tables/blobs/AssetsTable.h +++ b/testdata/golden/tables/blobs/AssetsTable.h @@ -4147,6 +4147,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace blobdemo #endif // BLOBDEMO_SCHEMA_TABLE_RETAIN @@ -4614,10 +5046,12 @@ template inline int64_t AssetMeasureBodyRetain( const Ctx & ctx, template inline bool AssetSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Asset & value, TableRetain * retain, const TableRetainPath & path ); template inline bool AssetSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Asset & value, TableRetain * retain, const TableRetainPath & path ); inline bool AssetLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Asset & value, TableRetain * retain, const TableRetainPath & path ); +inline bool AssetLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Asset & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t CatalogMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Catalog & value, TableRetain * retain, const TableRetainPath & path ); template inline bool CatalogSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Catalog & value, TableRetain * retain, const TableRetainPath & path ); template inline bool CatalogSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Catalog & value, TableRetain * retain, const TableRetainPath & path ); inline bool CatalogLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Catalog & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CatalogLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Catalog & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t AssetMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Asset & value ) @@ -8154,78 +8588,238 @@ inline bool AssetLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, As } } -template -inline int64_t CatalogMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Catalog & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool AssetLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Asset & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.name_length < 0 || value.name_length > 32 ) { return -1; } // storage invariant - if ( value.name_length > 0 ) { bytes += TableLebBytes( ids.ref( 0xc4bcadba8e631b86ull ) ) + 1 + TableLebBytes( (uint64_t) ( value.name_length ) ) + ( value.name_length ); } // name - { - const Asset * pointee_head = AssetAt( ctx, value.head ); // *Asset - // A POINTER RIDES AS A NODE INDEX (docs/SPEC-TABLES.md §3.1): the - // header and the index and nothing below it, because the pointee's - // body is in the node table and not here. NULL IS ELIDED — absence - // and null are one value — and a non-null pointer ALWAYS rides, even - // when its node's body is entirely default. - if ( pointee_head != NULL ) - { - uint64_t index_head = 0; - if ( !TableNumberingIndex( numbering, (const void *) pointee_head, index_head ) ) { return -1; } - bytes += TableLebBytes( ids.ref( 0x0a8f12cc5f9a0c03ull ) ) + 1 + TableLebBytes( index_head ); - } - } - { - const TableBlob * blob_thumb = TableBlobAt( ctx, value.thumb ); // *bytes - // A BYTE BUFFER RIDES AS A NODE INDEX too (docs/SPEC-TABLES.md §2.5, - // §3.1): the header and the index here, the bytes themselves as a - // record in the node table. Null is elided and a non-null blob always - // rides, even at length zero — null and empty are two values. - if ( blob_thumb != NULL ) - { - uint64_t index_thumb = 0; - if ( !TableNumberingIndex( numbering, (const void *) blob_thumb, index_thumb ) ) { return -1; } - bytes += TableLebBytes( ids.ref( 0x613b19720ff4b203ull ) ) + 1 + TableLebBytes( index_thumb ); - } - } - { - const TableBlob * blob_note = TableBlobAt( ctx, value.note ); // *string - // A BYTE BUFFER RIDES AS A NODE INDEX too (docs/SPEC-TABLES.md §2.5, - // §3.1): the header and the index here, the bytes themselves as a - // record in the node table. Null is elided and a non-null blob always - // rides, even at length zero — null and empty are two values. - if ( blob_note != NULL ) - { - uint64_t index_note = 0; - if ( !TableNumberingIndex( numbering, (const void *) blob_note, index_note ) ) { return -1; } - bytes += TableLebBytes( ids.ref( 0x3bf8fbbad1587cddull ) ) + 1 + TableLebBytes( index_note ); - } - } + (void) nodes; (void) index_bits; + AssetReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - const TableBlob * blob_alias = TableBlobAt( ctx, value.alias ); // *bytes - // A BYTE BUFFER RIDES AS A NODE INDEX too (docs/SPEC-TABLES.md §2.5, - // §3.1): the header and the index here, the bytes themselves as a - // record in the node table. Null is elided and a non-null blob always - // rides, even at length zero — null and empty are two values. - if ( blob_alias != NULL ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - uint64_t index_alias = 0; - if ( !TableNumberingIndex( numbering, (const void *) blob_alias, index_alias ) ) { return -1; } - bytes += TableLebBytes( ids.ref( 0x509220bb65a646b7ull ) ) + 1 + TableLebBytes( index_alias ); - } - } - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool CatalogSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Catalog & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( value.name_length < 0 || value.name_length > 32 ) { return false; } // storage invariant - if ( value.name_length > 0 ) - { - w.putleb( ids.ref( 0xc4bcadba8e631b86ull ) ); w.put8( 12 ); // name - w.putleb( (uint64_t) value.name_length ); - w.raw( value.name, value.name_length ); + case 0xc4bcadba8e631b86ull: // name + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 32 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.name, text, (size_t) kept ); + value.name[kept] = 0; + r.offset += (int64_t) n * 8; + value.name_length = kept; + } + break; + } + case 0xef9c96d721673243ull: // kind + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.kind = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.kind = decoded_v; + } + break; + } + case 0x855b556730a34a05ull: // data + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.data, node_index, kTableBytesTypeId, report ); // *bytes + } + break; + } + case 0x5daa28eb864c02a5ull: // caption + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.caption, node_index, kTableStringTypeId, report ); // *string + } + break; + } + case 0xe5316cbaa025f028ull: // next + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.next, node_index, 0x8e4e6dccfe5f64fbull, report ); // *Asset + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t CatalogMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Catalog & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.name_length < 0 || value.name_length > 32 ) { return -1; } // storage invariant + if ( value.name_length > 0 ) { bytes += TableLebBytes( ids.ref( 0xc4bcadba8e631b86ull ) ) + 1 + TableLebBytes( (uint64_t) ( value.name_length ) ) + ( value.name_length ); } // name + { + const Asset * pointee_head = AssetAt( ctx, value.head ); // *Asset + // A POINTER RIDES AS A NODE INDEX (docs/SPEC-TABLES.md §3.1): the + // header and the index and nothing below it, because the pointee's + // body is in the node table and not here. NULL IS ELIDED — absence + // and null are one value — and a non-null pointer ALWAYS rides, even + // when its node's body is entirely default. + if ( pointee_head != NULL ) + { + uint64_t index_head = 0; + if ( !TableNumberingIndex( numbering, (const void *) pointee_head, index_head ) ) { return -1; } + bytes += TableLebBytes( ids.ref( 0x0a8f12cc5f9a0c03ull ) ) + 1 + TableLebBytes( index_head ); + } + } + { + const TableBlob * blob_thumb = TableBlobAt( ctx, value.thumb ); // *bytes + // A BYTE BUFFER RIDES AS A NODE INDEX too (docs/SPEC-TABLES.md §2.5, + // §3.1): the header and the index here, the bytes themselves as a + // record in the node table. Null is elided and a non-null blob always + // rides, even at length zero — null and empty are two values. + if ( blob_thumb != NULL ) + { + uint64_t index_thumb = 0; + if ( !TableNumberingIndex( numbering, (const void *) blob_thumb, index_thumb ) ) { return -1; } + bytes += TableLebBytes( ids.ref( 0x613b19720ff4b203ull ) ) + 1 + TableLebBytes( index_thumb ); + } + } + { + const TableBlob * blob_note = TableBlobAt( ctx, value.note ); // *string + // A BYTE BUFFER RIDES AS A NODE INDEX too (docs/SPEC-TABLES.md §2.5, + // §3.1): the header and the index here, the bytes themselves as a + // record in the node table. Null is elided and a non-null blob always + // rides, even at length zero — null and empty are two values. + if ( blob_note != NULL ) + { + uint64_t index_note = 0; + if ( !TableNumberingIndex( numbering, (const void *) blob_note, index_note ) ) { return -1; } + bytes += TableLebBytes( ids.ref( 0x3bf8fbbad1587cddull ) ) + 1 + TableLebBytes( index_note ); + } + } + { + const TableBlob * blob_alias = TableBlobAt( ctx, value.alias ); // *bytes + // A BYTE BUFFER RIDES AS A NODE INDEX too (docs/SPEC-TABLES.md §2.5, + // §3.1): the header and the index here, the bytes themselves as a + // record in the node table. Null is elided and a non-null blob always + // rides, even at length zero — null and empty are two values. + if ( blob_alias != NULL ) + { + uint64_t index_alias = 0; + if ( !TableNumberingIndex( numbering, (const void *) blob_alias, index_alias ) ) { return -1; } + bytes += TableLebBytes( ids.ref( 0x509220bb65a646b7ull ) ) + 1 + TableLebBytes( index_alias ); + } + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool CatalogSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Catalog & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.name_length < 0 || value.name_length > 32 ) { return false; } // storage invariant + if ( value.name_length > 0 ) + { + w.putleb( ids.ref( 0xc4bcadba8e631b86ull ) ); w.put8( 12 ); // name + w.putleb( (uint64_t) value.name_length ); + w.raw( value.name, value.name_length ); } { const Asset * pointee_head = AssetAt( ctx, value.head ); // *Asset @@ -8427,6 +9021,146 @@ inline bool CatalogLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool CatalogLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Catalog & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + CatalogReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xc4bcadba8e631b86ull: // name + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 32 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.name, text, (size_t) kept ); + value.name[kept] = 0; + r.offset += (int64_t) n * 8; + value.name_length = kept; + } + break; + } + case 0x0a8f12cc5f9a0c03ull: // head + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.head, node_index, 0x8e4e6dccfe5f64fbull, report ); // *Asset + } + break; + } + case 0x613b19720ff4b203ull: // thumb + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.thumb, node_index, kTableBytesTypeId, report ); // *bytes + } + break; + } + case 0x3bf8fbbad1587cddull: // note + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.note, node_index, kTableStringTypeId, report ); // *string + } + break; + } + case 0x509220bb65a646b7ull: // alias + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.alias, node_index, kTableBytesTypeId, report ); // *bytes + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // AssetNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8587,6 +9321,161 @@ inline const Asset * AssetLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// AssetNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool AssetNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0x8e4e6dccfe5f64fbull: ok = AssetLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(Asset *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Asset + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// AssetLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool AssetLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Asset * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !AssetMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = AssetNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + AssetNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Asset ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x8e4e6dccfe5f64fbull; + Asset * root = new ( region + used ) Asset; // lifetime only: LoadMessageBody's first act is AssetReset + AssetReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !AssetNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return AssetLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// AssetLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool AssetLoadRetainMessages( const Asset ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !AssetLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // AssetMeasureRetain and AssetSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save @@ -8881,6 +9770,161 @@ inline const Catalog * CatalogLoadRetain( uint8_t * region, int64_t region_bytes return root; } +// CatalogNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool CatalogNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0x8e4e6dccfe5f64fbull: ok = AssetLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(Asset *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Asset + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// CatalogLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool CatalogLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Catalog * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !CatalogMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = CatalogNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + CatalogNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Catalog ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x1f1750c1bc916638ull; + Catalog * root = new ( region + used ) Catalog; // lifetime only: LoadMessageBody's first act is CatalogReset + CatalogReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !CatalogNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return CatalogLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// CatalogLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool CatalogLoadRetainMessages( const Catalog ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !CatalogLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // CatalogMeasureRetain and CatalogSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/lists/HoldersTable.h b/testdata/golden/tables/lists/HoldersTable.h index b15dcaff2..cdf77684f 100644 --- a/testdata/golden/tables/lists/HoldersTable.h +++ b/testdata/golden/tables/lists/HoldersTable.h @@ -4278,6 +4278,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace listdemo #endif // LISTDEMO_SCHEMA_TABLE_RETAIN @@ -6291,32 +6723,40 @@ inline SquadRosterEntryKeyRead SquadRosterEntryReadKey( const uint8_t * body, in inline int64_t SampleMeasureBodyRetain( TableRetainIds & ids, const Sample & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool SampleSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Sample & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool SampleLoadBodyRetain( TableReader & r, Sample & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SampleLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Sample & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t RowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Row & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RowSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Row & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RowSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Row & value, TableRetain * retain, const TableRetainPath & path ); inline bool RowLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Row & value, TableRetain * retain, const TableRetainPath & path ); +inline bool RowLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Row & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t SheetMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Sheet & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SheetSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Sheet & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SheetSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Sheet & value, TableRetain * retain, const TableRetainPath & path ); inline bool SheetLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Sheet & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SheetLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Sheet & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t ItemMeasureBodyRetain( TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool ItemSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool ItemLoadBodyRetain( TableReader & r, Item & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ItemLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Item & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t SquadRosterEntryMeasureBodyRetain( TableRetainIds & ids, const SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool SquadRosterEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool SquadRosterEntryLoadBodyRetain( TableReader & r, SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SquadRosterEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t SquadMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Squad & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SquadSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Squad & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SquadSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Squad & value, TableRetain * retain, const TableRetainPath & path ); inline bool SquadLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Squad & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SquadLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Squad & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t ArmyMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Army & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ArmySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Army & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ArmySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Army & value, TableRetain * retain, const TableRetainPath & path ); inline bool ArmyLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Army & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ArmyLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Army & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t DeckMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Deck & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DeckSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Deck & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DeckSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Deck & value, TableRetain * retain, const TableRetainPath & path ); inline bool DeckLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Deck & value, TableRetain * retain, const TableRetainPath & path ); +inline bool DeckLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Deck & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t SampleMeasureBody( TableIds & ids, const Sample & value ) { @@ -14803,6 +15243,98 @@ LISTDEMO_TABLE_INLINE bool SampleLoadBodyRetain( TableReader & r, Sample & value } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SampleLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Sample & value, TableRetain * retain, const TableRetainPath & path ) +{ + SampleReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xaf63eb4c86020609ull: // v + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.v = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.v = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t RowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Row & value, TableRetain * retain, const TableRetainPath & path ) { @@ -15014,11 +15546,135 @@ inline bool RowLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Row } } -template -inline int64_t SheetMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Sheet & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool RowLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Row & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + RowReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3e7884bf4f412c6full: // items + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableListFill fill = TableListFillBegin( nodes, value.items, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Sample * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + if ( !SampleLoadMessageBodyRetain( r, vocabulary, report, index_bits, ( *slot ), retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { return false; } + } + TableListFillEnd( fill ); + } + break; + } + case 0x39f7fcec8fcb623dull: // label + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.label = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.label = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t SheetMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Sheet & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { // rows: a kind 14 array of kind 13 elements, INDEX order (§2.9) TableListCursor cursor_rows = TableListElements( ctx, value.rows ); if ( !cursor_rows.ok ) { return -1; } // the slot and the head disagree @@ -15234,6 +15890,98 @@ inline bool SheetLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Sh } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SheetLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Sheet & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + SheetReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xa3a7061ff10a8138ull: // rows + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableListFill fill = TableListFillBegin( nodes, value.rows, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Row * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + if ( !RowLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( *slot ), retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { return false; } + } + TableListFillEnd( fill ); + } + break; + } + case 0x5f82477707ad620full: // pinned + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.pinned, node_index, 0xa013e119fec906fbull, report ); // *Row + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + inline int64_t ItemMeasureBodyRetain( TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ) { int64_t bytes = 1; // the ZERO REFERENCE that ends the body @@ -15320,63 +16068,155 @@ LISTDEMO_TABLE_INLINE bool ItemLoadBodyRetain( TableReader & r, Item & value, Ta } } -inline int64_t SquadRosterEntryMeasureBodyRetain( TableRetainIds & ids, const SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 1; } // key - { - const int32_t mark_value = ids.count; - const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); - const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); - if ( body_value < 0 ) { return -1; } - if ( body_value > 1 ) { bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); } // value - else { ids.truncate( mark_value ); } // an all-default nested table elides, and costs no entry - } - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -LISTDEMO_TABLE_INLINE bool SquadRosterEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( value.key != 0 ) - { - w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 6 ); // key - w.put8( uint8_t( value.key ) ); - } - { - const int32_t mark_value = ids.count; - const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); - const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); - if ( body_value < 0 ) return false; // storage invariant, refused as measure refuses it - if ( body_value > 1 ) // all-default nested elides - { - w.putleb( ref_value ); w.put8( 13 ); w.putleb( (uint64_t) body_value ); // value - if ( !ItemSaveBodyRetain( w, ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) return false; - } - else { ids.truncate( mark_value ); } - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -LISTDEMO_TABLE_INLINE bool SquadRosterEntryLoadBodyRetain( TableReader & r, SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ItemLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Item & value, TableRetain * retain, const TableRetainPath & path ) { - SquadRosterEntryReset( value ); // prefill declared defaults in place, then overlay + ItemReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count - const uint64_t field_id = r.ids->at( field_ref ); - if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } - uint8_t kind = r.get8(); + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xb1e5e28e4479a274ull: // count + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.count = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.count = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t SquadRosterEntryMeasureBodyRetain( TableRetainIds & ids, const SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 1; } // key + { + const int32_t mark_value = ids.count; + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); + if ( body_value < 0 ) { return -1; } + if ( body_value > 1 ) { bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); } // value + else { ids.truncate( mark_value ); } // an all-default nested table elides, and costs no entry + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +LISTDEMO_TABLE_INLINE bool SquadRosterEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 6 ); // key + w.put8( uint8_t( value.key ) ); + } + { + const int32_t mark_value = ids.count; + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); + if ( body_value < 0 ) return false; // storage invariant, refused as measure refuses it + if ( body_value > 1 ) // all-default nested elides + { + w.putleb( ref_value ); w.put8( 13 ); w.putleb( (uint64_t) body_value ); // value + if ( !ItemSaveBodyRetain( w, ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) return false; + } + else { ids.truncate( mark_value ); } + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +LISTDEMO_TABLE_INLINE bool SquadRosterEntryLoadBodyRetain( TableReader & r, SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + SquadRosterEntryReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) { // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, @@ -15440,6 +16280,86 @@ LISTDEMO_TABLE_INLINE bool SquadRosterEntryLoadBodyRetain( TableReader & r, Squa } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SquadRosterEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + SquadRosterEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 6 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 255ull ) { decoded_wide = (int64_t) 255ull; report->clamped++; } + uint8_t decoded_v = (uint8_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t SquadMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Squad & value, TableRetain * retain, const TableRetainPath & path ) { @@ -15676,47 +16596,205 @@ inline bool SquadLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Sq } } -template -inline int64_t ArmyMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Army & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // squads: a kind 14 array of kind 13 elements, INDEX order (§2.9) - TableListCursor cursor_squads = TableListElements( ctx, value.squads ); - if ( !cursor_squads.ok ) { return -1; } // the slot and the head disagree - if ( cursor_squads.count > 0 ) // an EMPTY list elides, the by-value rule (§3) - { - const uint64_t ref_squads = ids.ref( 0x7848019b0c02a926ull ); - int64_t body_squads = 0; - body_squads += 1 + TableLebBytes( (uint64_t) ( cursor_squads.count ) ); // the element kind byte and the count - for ( int32_t elem_i_squads = 0; elem_i_squads < cursor_squads.count; elem_i_squads++ ) - { - const int64_t elem_bytes_squads = SquadMeasureBodyRetain( ctx, numbering, ids, cursor_squads[elem_i_squads], retain, TableRetainStepInto( path, 0, (uint32_t) ( elem_i_squads ) ) ); - if ( elem_bytes_squads < 0 ) { return -1; } - body_squads += TableLebBytes( (uint64_t) ( elem_bytes_squads ) ) + ( elem_bytes_squads ); - } - bytes += TableLebBytes( ref_squads ) + 1 + TableLebBytes( (uint64_t) ( body_squads ) ) + ( body_squads ); - } - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool ArmySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Army & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SquadLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Squad & value, TableRetain * retain, const TableRetainPath & path ) { + (void) nodes; (void) index_bits; + SquadReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - TableListCursor cursor_squads = TableListElements( ctx, value.squads ); // squads - if ( !cursor_squads.ok ) { return false; } - if ( cursor_squads.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - const uint64_t ref_squads = ids.ref( 0x7848019b0c02a926ull ); - int64_t body_squads = 0; - body_squads += 1 + TableLebBytes( (uint64_t) ( cursor_squads.count ) ); // the element kind byte and the count - for ( int32_t elem_i_squads = 0; elem_i_squads < cursor_squads.count; elem_i_squads++ ) + case 0x1c84390d304f4f42ull: // roster { - const int64_t elem_bytes_squads = SquadMeasureBodyRetain( ctx, numbering, ids, cursor_squads[elem_i_squads], retain, TableRetainStepInto( path, 0, (uint32_t) ( elem_i_squads ) ) ); + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.roster, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint8_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const SquadRosterEntryMessageKeyRead read = SquadRosterEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + SquadRosterEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !SquadRosterEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xc4bcadba8e631b86ull: // name + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.name = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.name = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t ArmyMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Army & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // squads: a kind 14 array of kind 13 elements, INDEX order (§2.9) + TableListCursor cursor_squads = TableListElements( ctx, value.squads ); + if ( !cursor_squads.ok ) { return -1; } // the slot and the head disagree + if ( cursor_squads.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_squads = ids.ref( 0x7848019b0c02a926ull ); + int64_t body_squads = 0; + body_squads += 1 + TableLebBytes( (uint64_t) ( cursor_squads.count ) ); // the element kind byte and the count + for ( int32_t elem_i_squads = 0; elem_i_squads < cursor_squads.count; elem_i_squads++ ) + { + const int64_t elem_bytes_squads = SquadMeasureBodyRetain( ctx, numbering, ids, cursor_squads[elem_i_squads], retain, TableRetainStepInto( path, 0, (uint32_t) ( elem_i_squads ) ) ); + if ( elem_bytes_squads < 0 ) { return -1; } + body_squads += TableLebBytes( (uint64_t) ( elem_bytes_squads ) ) + ( elem_bytes_squads ); + } + bytes += TableLebBytes( ref_squads ) + 1 + TableLebBytes( (uint64_t) ( body_squads ) ) + ( body_squads ); + } + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool ArmySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Army & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableListCursor cursor_squads = TableListElements( ctx, value.squads ); // squads + if ( !cursor_squads.ok ) { return false; } + if ( cursor_squads.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_squads = ids.ref( 0x7848019b0c02a926ull ); + int64_t body_squads = 0; + body_squads += 1 + TableLebBytes( (uint64_t) ( cursor_squads.count ) ); // the element kind byte and the count + for ( int32_t elem_i_squads = 0; elem_i_squads < cursor_squads.count; elem_i_squads++ ) + { + const int64_t elem_bytes_squads = SquadMeasureBodyRetain( ctx, numbering, ids, cursor_squads[elem_i_squads], retain, TableRetainStepInto( path, 0, (uint32_t) ( elem_i_squads ) ) ); if ( elem_bytes_squads < 0 ) { return false; } body_squads += TableLebBytes( (uint64_t) ( elem_bytes_squads ) ) + ( elem_bytes_squads ); } @@ -15884,6 +16962,130 @@ inline bool ArmyLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Arm } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ArmyLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Army & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + ArmyReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x7848019b0c02a926ull: // squads + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableListFill fill = TableListFillBegin( nodes, value.squads, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Squad * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + if ( !SquadLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( *slot ), retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { return false; } + } + TableListFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t DeckMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Deck & value, TableRetain * retain, const TableRetainPath & path ) { @@ -16058,22 +17260,150 @@ inline bool DeckLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Dec if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } break; } - if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } - int32_t decoded_v = int32_t( r.get32( ) ); - value.after = decoded_v; - break; - } - case 0xffffffffffffffffull: - { - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool DeckLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Deck & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + DeckReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x81b46a69304ee2c9ull: // hands + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 3 ) { kept = 3; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + Row scratch; + RowReset( scratch ); + if ( !RowLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( in_bounds ? value.hands[i] : scratch ), ( in_bounds ? retain : NULL ), TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { return false; } + } + value.hands_count = kept; + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } break; } default: + report->unknown++; { - r.report->unknown++; - if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } - break; + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); } + break; } } } @@ -16251,6 +17581,182 @@ inline const Row * RowLoadRetain( uint8_t * region, int64_t region_bytes, const return root; } +// RowNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool RowNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + RowNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !RowNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// RowLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool RowLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Row * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !RowMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = RowNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + RowNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !RowMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Row ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xa013e119fec906fbull; + Row * root = new ( region + used ) Row; // lifetime only: LoadMessageBody's first act is RowReset + RowReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Row ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !RowNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return RowLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// RowLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool RowLoadRetainMessages( const Row ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !RowLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // RowMeasureRetain and RowSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save @@ -16509,33 +18015,208 @@ inline const Sheet * SheetLoadRetain( uint8_t * region, int64_t region_bytes, co if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } } - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + SheetNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Sheet ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + SheetLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// SheetNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool SheetNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + SheetNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !SheetNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + bool ok = false; + switch ( type_id ) + { + case 0xa013e119fec906fbull: ok = RowLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(Row *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Row + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// SheetLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool SheetLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Sheet * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !SheetMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = SheetNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + SheetNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !SheetMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Sheet ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x0cc9e0af9a85fbc8ull; + Sheet * root = new ( region + used ) Sheet; // lifetime only: LoadMessageBody's first act is SheetReset + SheetReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Sheet ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - SheetNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !SheetNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Sheet ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - SheetLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return SheetLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// SheetLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool SheetLoadRetainMessages( const Sheet ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !SheetLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // SheetMeasureRetain and SheetSaveRetain: the pair, with the retained tail in @@ -16762,71 +18443,247 @@ inline const Squad * SquadLoadRetain( uint8_t * region, int64_t region_bytes, co // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Squad ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = SquadNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + SquadNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + SquadNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Squad ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + SquadLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// SquadNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool SquadNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + SquadNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !SquadNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// SquadLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool SquadLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Squad * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !SquadMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = SquadNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + SquadNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !SquadMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Squad ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xec07a2f760550a91ull; + Squad * root = new ( region + used ) Squad; // lifetime only: LoadMessageBody's first act is SquadReset + SquadReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Squad ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Squad ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = SquadNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - SquadNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - SquadNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !SquadNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Squad ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - SquadLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return SquadLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// SquadLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool SquadLoadRetainMessages( const Squad ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !SquadLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // SquadMeasureRetain and SquadSaveRetain: the pair, with the retained tail in @@ -17050,71 +18907,247 @@ inline const Army * ArmyLoadRetain( uint8_t * region, int64_t region_bytes, cons // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Army ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = ArmyNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + ArmyNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + ArmyNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Army ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + ArmyLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// ArmyNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool ArmyNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + ArmyNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !ArmyNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// ArmyLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool ArmyLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Army * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !ArmyMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = ArmyNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + ArmyNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !ArmyMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Army ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x06e2378b553a9e84ull; + Army * root = new ( region + used ) Army; // lifetime only: LoadMessageBody's first act is ArmyReset + ArmyReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Army ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Army ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = ArmyNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - ArmyNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - ArmyNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !ArmyNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Army ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - ArmyLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return ArmyLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// ArmyLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool ArmyLoadRetainMessages( const Army ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !ArmyLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // ArmyMeasureRetain and ArmySaveRetain: the pair, with the retained tail in @@ -17405,6 +19438,182 @@ inline const Deck * DeckLoadRetain( uint8_t * region, int64_t region_bytes, cons return root; } +// DeckNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool DeckNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + DeckNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !DeckNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// DeckLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool DeckLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Deck * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !DeckMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = DeckNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + DeckNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !DeckMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Deck ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xd043187343bfcfe8ull; + Deck * root = new ( region + used ) Deck; // lifetime only: LoadMessageBody's first act is DeckReset + DeckReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Deck ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !DeckNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return DeckLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// DeckLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool DeckLoadRetainMessages( const Deck ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !DeckLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // DeckMeasureRetain and DeckSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/lists/MigrateTable.h b/testdata/golden/tables/lists/MigrateTable.h index b2e69494f..673fb6f5c 100644 --- a/testdata/golden/tables/lists/MigrateTable.h +++ b/testdata/golden/tables/lists/MigrateTable.h @@ -4278,6 +4278,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace listdemo #endif // LISTDEMO_SCHEMA_TABLE_RETAIN @@ -5997,13 +6429,16 @@ template inline bool TableNodeMessageSave( const Ctx & ctx, const inline int64_t UnitMeasureBodyRetain( TableRetainIds & ids, const Unit & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool UnitSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Unit & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool UnitLoadBodyRetain( TableReader & r, Unit & value, TableRetain * retain, const TableRetainPath & path ); +inline bool UnitLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Unit & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t BoundedMeasureBodyRetain( TableRetainIds & ids, const Bounded & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool BoundedSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Bounded & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool BoundedLoadBodyRetain( TableReader & r, Bounded & value, TableRetain * retain, const TableRetainPath & path ); +inline bool BoundedLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Bounded & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t UnboundedMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Unbounded & value, TableRetain * retain, const TableRetainPath & path ); template inline bool UnboundedSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Unbounded & value, TableRetain * retain, const TableRetainPath & path ); template inline bool UnboundedSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Unbounded & value, TableRetain * retain, const TableRetainPath & path ); inline bool UnboundedLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Unbounded & value, TableRetain * retain, const TableRetainPath & path ); +inline bool UnboundedLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Unbounded & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t UnitMeasureBody( TableIds & ids, const Unit & value ) { @@ -8358,6 +8793,98 @@ LISTDEMO_TABLE_INLINE bool UnitLoadBodyRetain( TableReader & r, Unit & value, Ta } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool UnitLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Unit & value, TableRetain * retain, const TableRetainPath & path ) +{ + UnitReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xaf63eb4c86020609ull: // v + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.v = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.v = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + inline int64_t BoundedMeasureBodyRetain( TableRetainIds & ids, const Bounded & value, TableRetain * retain, const TableRetainPath & path ) { int64_t bytes = 1; // the ZERO REFERENCE that ends the body @@ -8538,10 +9065,137 @@ LISTDEMO_TABLE_INLINE bool BoundedLoadBodyRetain( TableReader & r, Bounded & val } } -template -inline int64_t UnboundedMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Unbounded & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool BoundedLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Bounded & value, TableRetain * retain, const TableRetainPath & path ) { - (void) ctx; (void) numbering; + BoundedReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3e7884bf4f412c6full: // items + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 8 ) { kept = 8; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + Unit scratch; + UnitReset( scratch ); + if ( !UnitLoadMessageBodyRetain( r, vocabulary, report, index_bits, ( in_bounds ? value.items[i] : scratch ), ( in_bounds ? retain : NULL ), TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { return false; } + } + value.items_count = kept; + } + break; + } + case 0x56d7ab194448a4f3ull: // tag + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.tag = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.tag = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t UnboundedMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Unbounded & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; int64_t bytes = 1; // the ZERO REFERENCE that ends the body { // items: a kind 14 array of kind 13 elements, INDEX order (§2.9) @@ -8749,6 +9403,130 @@ inline bool UnboundedLoadBodyRetain( TableReader & r, const TableNodeMap & nodes } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool UnboundedLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Unbounded & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + UnboundedReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3e7884bf4f412c6full: // items + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableListFill fill = TableListFillBegin( nodes, value.items, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Unit * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + if ( !UnitLoadMessageBodyRetain( r, vocabulary, report, index_bits, ( *slot ), retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { return false; } + } + TableListFillEnd( fill ); + } + break; + } + case 0x56d7ab194448a4f3ull: // tag + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.tag = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.tag = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // UnboundedNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8922,6 +9700,182 @@ inline const Unbounded * UnboundedLoadRetain( uint8_t * region, int64_t region_b return root; } +// UnboundedNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool UnboundedNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + UnboundedNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !UnboundedNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// UnboundedLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool UnboundedLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Unbounded * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !UnboundedMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = UnboundedNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + UnboundedNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !UnboundedMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Unbounded ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x2ec2f34386026d8bull; + Unbounded * root = new ( region + used ) Unbounded; // lifetime only: LoadMessageBody's first act is UnboundedReset + UnboundedReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Unbounded ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !UnboundedNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return UnboundedLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// UnboundedLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool UnboundedLoadRetainMessages( const Unbounded ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !UnboundedLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // UnboundedMeasureRetain and UnboundedSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/lists/ReportTable.h b/testdata/golden/tables/lists/ReportTable.h index 04421138a..272dd543f 100644 --- a/testdata/golden/tables/lists/ReportTable.h +++ b/testdata/golden/tables/lists/ReportTable.h @@ -4278,6 +4278,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace listdemo #endif // LISTDEMO_SCHEMA_TABLE_RETAIN @@ -6019,14 +6451,17 @@ template inline int64_t BytesMeasureBodyRetain( const Ctx & ctx, template inline bool BytesSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Bytes & value, TableRetain * retain, const TableRetainPath & path ); template inline bool BytesSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Bytes & value, TableRetain * retain, const TableRetainPath & path ); inline bool BytesLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Bytes & value, TableRetain * retain, const TableRetainPath & path ); +inline bool BytesLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Bytes & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t IntsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ); template inline bool IntsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ); template inline bool IntsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ); inline bool IntsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Ints & value, TableRetain * retain, const TableRetainPath & path ); +inline bool IntsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Ints & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t FloatsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FloatsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FloatsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ); inline bool FloatsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Floats & value, TableRetain * retain, const TableRetainPath & path ); +inline bool FloatsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Floats & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t BytesMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Bytes & value ) @@ -10589,233 +11024,145 @@ inline bool BytesLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, By } } -template -inline int64_t IntsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ) -{ - (void) ctx; (void) numbering; - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // values: a kind 14 array of kind 4 elements, INDEX order (§2.9) - TableListCursor cursor_values = TableListElements( ctx, value.values ); - if ( !cursor_values.ok ) { return -1; } // the slot and the head disagree - if ( cursor_values.count > 0 ) // an EMPTY list elides, the by-value rule (§3) - { - const uint64_t ref_values = ids.ref( 0x21277bcf1a4d67fbull ); - int64_t body_values = 0; - body_values += 1 + TableLebBytes( (uint64_t) ( cursor_values.count ) ); // the element kind byte and the count - body_values += (int64_t) ( cursor_values.count ) * 4; - bytes += TableLebBytes( ref_values ) + 1 + TableLebBytes( (uint64_t) ( body_values ) ) + ( body_values ); - } - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool IntsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ) -{ - (void) ctx; (void) numbering; - { - TableListCursor cursor_values = TableListElements( ctx, value.values ); // values - if ( !cursor_values.ok ) { return false; } - if ( cursor_values.count > 0 ) // an EMPTY list elides, the by-value rule (§3) - { - const uint64_t ref_values = ids.ref( 0x21277bcf1a4d67fbull ); - int64_t body_values = 0; - body_values += 1 + TableLebBytes( (uint64_t) ( cursor_values.count ) ); // the element kind byte and the count - body_values += (int64_t) ( cursor_values.count ) * 4; - w.putleb( ref_values ); w.put8( 14 ); w.putleb( (uint64_t) body_values ); // values - w.put8( 4 ); w.putleb( (uint64_t) ( cursor_values.count ) ); - for ( int32_t elem_i_values = 0; elem_i_values < cursor_values.count; elem_i_values++ ) - { - w.put32( uint32_t( cursor_values[elem_i_values] ) ); - } - } - } - if ( value.after != 0 ) - { - w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after - w.put32( uint32_t( value.after ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - return !w.overflow; -} - -template -inline bool IntsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( !IntsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -inline bool IntsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Ints & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool BytesLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Bytes & value, TableRetain * retain, const TableRetainPath & path ) { - (void) nodes; - IntsReset( value ); // prefill declared defaults in place, then overlay + (void) nodes; (void) index_bits; + BytesReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count - const uint64_t field_id = r.ids->at( field_ref ); - if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } - uint8_t kind = r.get8(); - if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) - { - // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, - // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node - // table's is the ROOT body's alone, on the numbering's own - // rule — a second numbering cannot exist — and the BUILD - // VERSION's rides in the announcement and nowhere else. That - // body stops and the parent reads on past its L. - r.report->malformed = true; - return false; - } - switch ( field_id ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - case 0x21277bcf1a4d67fbull: // values + case 0x855b556730a34a05ull: // data { - if ( kind != 14 ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 6 ) { - // AT A POSITION THE READER DOES NAME, a field under - // kind 31 or kind 32 takes this same rule and no other (§3) - r.report->kind_mismatch++; - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } break; } - uint64_t body_len = 0; - if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } - int64_t body_end = r.offset + (int64_t) body_len; - // A BODY TOO SHORT FOR ITS OWN HEADER is INERT (§4): the field keeps - // the value it has, no counter is raised, and the walk continues past L. - if ( body_len >= 2 ) { - uint8_t elem_kind = r.get8(); - uint64_t count = 0; - const bool counted_ok = r.getleb( count ); - if ( !counted_ok ) { r.report->malformed = true; } - // AN ELEMENT KIND THAT DISAGREES with the reader's declaration is §3's - // element-kind rule: the field reads EMPTY and one kind_mismatch counts - else if ( elem_kind != 4 ) - { - if ( !TableKindWidens( elem_kind, 4 ) ) { r.report->kind_mismatch++; r.offset = body_end; break; } - r.report->widened++; - // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped - // cannot fire on it. A count above the int32 storage cap is the - // fill's refusal, and it moves no counter. - TableListFill fill = TableListFillBegin( nodes, value.values, count ); - if ( fill.refused ) { nodes.refused = true; return false; } - if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } - // elements are BOUNDED by the field body: a count the length cannot - // cover keeps the decoded prefix, flags malformed, and the parent - // continues at the next field - TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); - for ( uint64_t i = 0; i < count; i++ ) - { - int32_t * slot = TableListFillNext( fill ); - if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve - bool landed = false; - do - { - int64_t widened_v = 0; - if ( !TableReadSignedAt( sub, elem_kind, widened_v ) ) { r.report->malformed = true; break; } - int32_t decoded_v = (int32_t) widened_v; - ( *slot ) = decoded_v; - landed = true; - } while ( 0 ); - if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded - } - TableListFillEnd( fill ); - } - else + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + if ( !r.align() ) { report->malformed = true; return false; } // an array of kind 6 aligns before its elements + TableListFill fill = TableListFillBegin( nodes, value.data, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) { - // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped - // cannot fire on it. A count above the int32 storage cap is the - // fill's refusal, and it moves no counter. - TableListFill fill = TableListFillBegin( nodes, value.values, count ); - if ( fill.refused ) { nodes.refused = true; return false; } - if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } - // elements are BOUNDED by the field body: a count the length cannot - // cover keeps the decoded prefix, flags malformed, and the parent - // continues at the next field - TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); - for ( uint64_t i = 0; i < count; i++ ) + uint8_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve { - int32_t * slot = TableListFillNext( fill ); - if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve - bool landed = false; - do - { - if ( !sub.has( 4 ) ) { r.report->malformed = true; break; } - int32_t decoded_v = int32_t( sub.get32( ) ); - ( *slot ) = decoded_v; - landed = true; - } while ( 0 ); - if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + if ( (uint64_t) decoded_wide_2 > 255ull ) { decoded_wide_2 = (int64_t) 255ull; report->clamped++; } + uint8_t decoded_v_2 = (uint8_t) decoded_wide_2; + ( *slot ) = decoded_v_2; } - TableListFillEnd( fill ); } + TableListFillEnd( fill ); } - r.offset = body_end; // excess bytes and slack skip via the length break; } case 0xbf82010f6f71eae9ull: // after { - if ( kind != 4 ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) { - if ( TableKindWidens( kind, 4 ) ) + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) { - // WIDENED (§4): a kind that grew since the writer decodes - // exactly at its own width, the value lands, one widened counts - int64_t widened_v = 0; - if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } - int32_t decoded_v = (int32_t) widened_v; - value.after = decoded_v; - r.report->widened++; + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; break; } - // AT A POSITION THE READER DOES NAME, a field under - // kind 31 or kind 32 takes this same rule and no other (§3) - r.report->kind_mismatch++; - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } break; } - if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } - int32_t decoded_v = int32_t( r.get32( ) ); - value.after = decoded_v; - break; - } - case 0xffffffffffffffffull: - { - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } break; } default: + report->unknown++; { - r.report->unknown++; - if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } - break; + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); } + break; } } } template -inline int64_t FloatsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ) +inline int64_t IntsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ) { (void) ctx; (void) numbering; int64_t bytes = 1; // the ZERO REFERENCE that ends the body { - // values: a kind 14 array of kind 10 elements, INDEX order (§2.9) - TableListCursor cursor_values = TableListElements( ctx, value.values ); + // values: a kind 14 array of kind 4 elements, INDEX order (§2.9) + TableListCursor cursor_values = TableListElements( ctx, value.values ); if ( !cursor_values.ok ) { return -1; } // the slot and the head disagree if ( cursor_values.count > 0 ) // an EMPTY list elides, the by-value rule (§3) { @@ -10832,11 +11179,11 @@ inline int64_t FloatsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & } template -inline bool FloatsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ) +inline bool IntsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ) { (void) ctx; (void) numbering; { - TableListCursor cursor_values = TableListElements( ctx, value.values ); // values + TableListCursor cursor_values = TableListElements( ctx, value.values ); // values if ( !cursor_values.ok ) { return false; } if ( cursor_values.count > 0 ) // an EMPTY list elides, the by-value rule (§3) { @@ -10845,10 +11192,10 @@ inline bool FloatsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & body_values += 1 + TableLebBytes( (uint64_t) ( cursor_values.count ) ); // the element kind byte and the count body_values += (int64_t) ( cursor_values.count ) * 4; w.putleb( ref_values ); w.put8( 14 ); w.putleb( (uint64_t) body_values ); // values - w.put8( 10 ); w.putleb( (uint64_t) ( cursor_values.count ) ); + w.put8( 4 ); w.putleb( (uint64_t) ( cursor_values.count ) ); for ( int32_t elem_i_values = 0; elem_i_values < cursor_values.count; elem_i_values++ ) { - w.put32( table_float_to_bits( cursor_values[elem_i_values] ) ); + w.put32( uint32_t( cursor_values[elem_i_values] ) ); } } } @@ -10862,17 +11209,17 @@ inline bool FloatsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & } template -inline bool FloatsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ) +inline bool IntsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Ints & value, TableRetain * retain, const TableRetainPath & path ) { - if ( !FloatsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + if ( !IntsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } w.put8( 0 ); // the ZERO REFERENCE that ends the body return !w.overflow; } -inline bool FloatsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Floats & value, TableRetain * retain, const TableRetainPath & path ) +inline bool IntsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Ints & value, TableRetain * retain, const TableRetainPath & path ) { (void) nodes; - FloatsReset( value ); // prefill declared defaults in place, then overlay + IntsReset( value ); // prefill declared defaults in place, then overlay // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT // (docs/SPEC-TABLES.md §6.6): this body is being established, so // whatever an earlier occurrence of it left is discarded before the @@ -10923,13 +11270,43 @@ inline bool FloatsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, F if ( !counted_ok ) { r.report->malformed = true; } // AN ELEMENT KIND THAT DISAGREES with the reader's declaration is §3's // element-kind rule: the field reads EMPTY and one kind_mismatch counts - else if ( elem_kind != 10 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else if ( elem_kind != 4 ) + { + if ( !TableKindWidens( elem_kind, 4 ) ) { r.report->kind_mismatch++; r.offset = body_end; break; } + r.report->widened++; + // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped + // cannot fire on it. A count above the int32 storage cap is the + // fill's refusal, and it moves no counter. + TableListFill fill = TableListFillBegin( nodes, value.values, count ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + // elements are BOUNDED by the field body: a count the length cannot + // cover keeps the decoded prefix, flags malformed, and the parent + // continues at the next field + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < count; i++ ) + { + int32_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve + bool landed = false; + do + { + int64_t widened_v = 0; + if ( !TableReadSignedAt( sub, elem_kind, widened_v ) ) { r.report->malformed = true; break; } + int32_t decoded_v = (int32_t) widened_v; + ( *slot ) = decoded_v; + landed = true; + } while ( 0 ); + if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + } + TableListFillEnd( fill ); + } else { // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped // cannot fire on it. A count above the int32 storage cap is the // fill's refusal, and it moves no counter. - TableListFill fill = TableListFillBegin( nodes, value.values, count ); + TableListFill fill = TableListFillBegin( nodes, value.values, count ); if ( fill.refused ) { nodes.refused = true; return false; } if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } // elements are BOUNDED by the field body: a count the length cannot @@ -10938,13 +11315,14 @@ inline bool FloatsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, F TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); for ( uint64_t i = 0; i < count; i++ ) { - float * slot = TableListFillNext( fill ); + int32_t * slot = TableListFillNext( fill ); if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve bool landed = false; do { if ( !sub.has( 4 ) ) { r.report->malformed = true; break; } - ( *slot ) = table_bits_to_float( sub.get32() ); + int32_t decoded_v = int32_t( sub.get32( ) ); + ( *slot ) = decoded_v; landed = true; } while ( 0 ); if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded @@ -10996,15 +11374,476 @@ inline bool FloatsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, F } } -// BytesNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void BytesNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool IntsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Ints & value, TableRetain * retain, const TableRetainPath & path ) { - // the node's own EXTENT, where its lists' and maps' arrays are carved - // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + (void) nodes; (void) index_bits; + IntsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x21277bcf1a4d67fbull: // values + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 4 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.values, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + int32_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + ( *slot ) = decoded_v_2; + } + } + TableListFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t FloatsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // values: a kind 14 array of kind 10 elements, INDEX order (§2.9) + TableListCursor cursor_values = TableListElements( ctx, value.values ); + if ( !cursor_values.ok ) { return -1; } // the slot and the head disagree + if ( cursor_values.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_values = ids.ref( 0x21277bcf1a4d67fbull ); + int64_t body_values = 0; + body_values += 1 + TableLebBytes( (uint64_t) ( cursor_values.count ) ); // the element kind byte and the count + body_values += (int64_t) ( cursor_values.count ) * 4; + bytes += TableLebBytes( ref_values ) + 1 + TableLebBytes( (uint64_t) ( body_values ) ) + ( body_values ); + } + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool FloatsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + { + TableListCursor cursor_values = TableListElements( ctx, value.values ); // values + if ( !cursor_values.ok ) { return false; } + if ( cursor_values.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_values = ids.ref( 0x21277bcf1a4d67fbull ); + int64_t body_values = 0; + body_values += 1 + TableLebBytes( (uint64_t) ( cursor_values.count ) ); // the element kind byte and the count + body_values += (int64_t) ( cursor_values.count ) * 4; + w.putleb( ref_values ); w.put8( 14 ); w.putleb( (uint64_t) body_values ); // values + w.put8( 10 ); w.putleb( (uint64_t) ( cursor_values.count ) ); + for ( int32_t elem_i_values = 0; elem_i_values < cursor_values.count; elem_i_values++ ) + { + w.put32( table_float_to_bits( cursor_values[elem_i_values] ) ); + } + } + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool FloatsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Floats & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !FloatsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool FloatsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Floats & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; + FloatsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x21277bcf1a4d67fbull: // values + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER is INERT (§4): the field keeps + // the value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + if ( !counted_ok ) { r.report->malformed = true; } + // AN ELEMENT KIND THAT DISAGREES with the reader's declaration is §3's + // element-kind rule: the field reads EMPTY and one kind_mismatch counts + else if ( elem_kind != 10 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped + // cannot fire on it. A count above the int32 storage cap is the + // fill's refusal, and it moves no counter. + TableListFill fill = TableListFillBegin( nodes, value.values, count ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + // elements are BOUNDED by the field body: a count the length cannot + // cover keeps the decoded prefix, flags malformed, and the parent + // continues at the next field + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < count; i++ ) + { + float * slot = TableListFillNext( fill ); + if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve + bool landed = false; + do + { + if ( !sub.has( 4 ) ) { r.report->malformed = true; break; } + ( *slot ) = table_bits_to_float( sub.get32() ); + landed = true; + } while ( 0 ); + if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + } + TableListFillEnd( fill ); + } + } + r.offset = body_end; // excess bytes and slack skip via the length + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool FloatsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Floats & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + FloatsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x21277bcf1a4d67fbull: // values + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 10 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.values, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + float * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + float decoded_f_2 = 0.0f; + if ( entry.elem_packing == 2 ) + { + uint64_t index_2 = 0; + if ( !r.get( index_2, entry.elem_value_bits ) ) { report->malformed = true; return false; } + if ( index_2 > (uint64_t) entry.elem_qcount ) { report->malformed = true; return false; } // above the step count: the packet wire's own refusal + decoded_f_2 = TableMessageDequantize( (uint32_t) index_2, entry.elem_qmin, entry.elem_qdelta, entry.elem_qcount ); // SPEC.md §4.3's rule, in float32 + } + else + { + uint64_t raw_2 = 0; + if ( !r.get( raw_2, 32 ) ) { report->malformed = true; return false; } + decoded_f_2 = table_bits_to_float( (uint32_t) raw_2 ); + } + ( *slot ) = decoded_f_2; + } + } + TableListFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// BytesNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void BytesNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). // The tool's path carries a worker instead: there the arrays are the // arena's. TableExtentCarve carve; @@ -11102,71 +11941,247 @@ inline const Bytes * BytesLoadRetain( uint8_t * region, int64_t region_bytes, co // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Bytes ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = BytesNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + BytesNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + BytesNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Bytes ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + BytesLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// BytesNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool BytesNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + BytesNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !BytesNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// BytesLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool BytesLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Bytes * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !BytesMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = BytesNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + BytesNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !BytesMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Bytes ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xeeeea7adc131a244ull; + Bytes * root = new ( region + used ) Bytes; // lifetime only: LoadMessageBody's first act is BytesReset + BytesReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Bytes ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Bytes ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = BytesNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - BytesNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - BytesNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !BytesNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Bytes ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - BytesLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return BytesLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// BytesLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool BytesLoadRetainMessages( const Bytes ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !BytesLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // BytesMeasureRetain and BytesSaveRetain: the pair, with the retained tail in @@ -11390,71 +12405,247 @@ inline const Ints * IntsLoadRetain( uint8_t * region, int64_t region_bytes, cons // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Ints ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = IntsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + IntsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + IntsNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Ints ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + IntsLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// IntsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool IntsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + IntsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !IntsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// IntsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool IntsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Ints * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !IntsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = IntsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + IntsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !IntsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Ints ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x2034c5d17c00ceb7ull; + Ints * root = new ( region + used ) Ints; // lifetime only: LoadMessageBody's first act is IntsReset + IntsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Ints ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Ints ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = IntsNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - IntsNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - IntsNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !IntsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Ints ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - IntsLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return IntsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// IntsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool IntsLoadRetainMessages( const Ints ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !IntsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // IntsMeasureRetain and IntsSaveRetain: the pair, with the retained tail in @@ -11745,6 +12936,182 @@ inline const Floats * FloatsLoadRetain( uint8_t * region, int64_t region_bytes, return root; } +// FloatsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool FloatsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + FloatsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !FloatsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// FloatsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool FloatsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Floats * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !FloatsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = FloatsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + FloatsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !FloatsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Floats ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x91638659f8f6ad42ull; + Floats * root = new ( region + used ) Floats; // lifetime only: LoadMessageBody's first act is FloatsReset + FloatsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Floats ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !FloatsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return FloatsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// FloatsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool FloatsLoadRetainMessages( const Floats ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !FloatsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // FloatsMeasureRetain and FloatsSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/lists/SaveTable.h b/testdata/golden/tables/lists/SaveTable.h index a3e1b541f..e39a25a5a 100644 --- a/testdata/golden/tables/lists/SaveTable.h +++ b/testdata/golden/tables/lists/SaveTable.h @@ -4278,6 +4278,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace listdemo #endif // LISTDEMO_SCHEMA_TABLE_RETAIN @@ -6220,20 +6652,25 @@ template inline bool TableNodeMessageSave( const Ctx & ctx, const inline int64_t PlacementMeasureBodyRetain( TableRetainIds & ids, const Placement & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool PlacementSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Placement & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool PlacementLoadBodyRetain( TableReader & r, Placement & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PlacementLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Placement & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t LogEntryMeasureBodyRetain( TableRetainIds & ids, const LogEntry & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool LogEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const LogEntry & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool LogEntryLoadBodyRetain( TableReader & r, LogEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool LogEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, LogEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t SaveMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Save & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SaveSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Save & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SaveSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Save & value, TableRetain * retain, const TableRetainPath & path ); inline bool SaveLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Save & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SaveLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Save & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t PointMeasureBodyRetain( TableRetainIds & ids, const Point & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool PointSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Point & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool PointLoadBodyRetain( TableReader & r, Point & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PointLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Point & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t MixedMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Mixed & value, TableRetain * retain, const TableRetainPath & path ); template inline bool MixedSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Mixed & value, TableRetain * retain, const TableRetainPath & path ); template inline bool MixedSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Mixed & value, TableRetain * retain, const TableRetainPath & path ); inline bool MixedLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Mixed & value, TableRetain * retain, const TableRetainPath & path ); +inline bool MixedLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Mixed & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t PlacementMeasureBody( TableIds & ids, const Placement & value ) { @@ -11922,80 +12359,220 @@ LISTDEMO_TABLE_INLINE bool PlacementLoadBodyRetain( TableReader & r, Placement & } } -inline int64_t LogEntryMeasureBodyRetain( TableRetainIds & ids, const LogEntry & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.tick != 0 ) { bytes += TableLebBytes( ids.ref( 0x1e7683ef2ebc7684ull ) ) + 1 + 4; } // tick - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -LISTDEMO_TABLE_INLINE bool LogEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const LogEntry & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( value.tick != 0 ) - { - w.putleb( ids.ref( 0x1e7683ef2ebc7684ull ) ); w.put8( 8 ); // tick - w.put32( uint32_t( value.tick ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -LISTDEMO_TABLE_INLINE bool LogEntryLoadBodyRetain( TableReader & r, LogEntry & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool PlacementLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Placement & value, TableRetain * retain, const TableRetainPath & path ) { - LogEntryReset( value ); // prefill declared defaults in place, then overlay + PlacementReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count - const uint64_t field_id = r.ids->at( field_ref ); - if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } - uint8_t kind = r.get8(); - if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) - { - // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, - // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node - // table's is the ROOT body's alone, on the numbering's own - // rule — a second numbering cannot exist — and the BUILD - // VERSION's rides in the announcement and nowhere else. That - // body stops and the parent reads on past its L. - r.report->malformed = true; - return false; - } - switch ( field_id ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - case 0x1e7683ef2ebc7684ull: // tick + case 0xaf63f54c86021707ull: // x { - if ( kind != 8 ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 10 || entry.elem_kind != 0 ) { - if ( TableKindWidens( kind, 8 ) ) - { - // WIDENED (§4): a kind that grew since the writer decodes - // exactly at its own width, the value lands, one widened counts - uint64_t widened_v = 0; - if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } - uint32_t decoded_v = (uint32_t) widened_v; - value.tick = decoded_v; - r.report->widened++; - break; - } - // AT A POSITION THE READER DOES NAME, a field under - // kind 31 or kind 32 takes this same rule and no other (§3) - r.report->kind_mismatch++; - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } break; } - if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } - uint32_t decoded_v = uint32_t( r.get32( ) ); - value.tick = decoded_v; + { + float decoded_f = 0.0f; + if ( entry.packing == 2 ) + { + uint64_t index = 0; + if ( !r.get( index, entry.value_bits ) ) { report->malformed = true; return false; } + if ( index > (uint64_t) entry.qcount ) { report->malformed = true; return false; } // above the step count: the packet wire's own refusal + decoded_f = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); // SPEC.md §4.3's rule, in float32 + } + else + { + uint64_t raw = 0; + if ( !r.get( raw, 32 ) ) { report->malformed = true; return false; } + decoded_f = table_bits_to_float( (uint32_t) raw ); + } + value.x = decoded_f; + } + break; + } + case 0xaf63f44c86021554ull: // y + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 10 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + float decoded_f = 0.0f; + if ( entry.packing == 2 ) + { + uint64_t index = 0; + if ( !r.get( index, entry.value_bits ) ) { report->malformed = true; return false; } + if ( index > (uint64_t) entry.qcount ) { report->malformed = true; return false; } // above the step count: the packet wire's own refusal + decoded_f = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); // SPEC.md §4.3's rule, in float32 + } + else + { + uint64_t raw = 0; + if ( !r.get( raw, 32 ) ) { report->malformed = true; return false; } + decoded_f = table_bits_to_float( (uint32_t) raw ); + } + value.y = decoded_f; + } + break; + } + case 0x9de543933e6e703aull: // model + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.model = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.model = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t LogEntryMeasureBodyRetain( TableRetainIds & ids, const LogEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.tick != 0 ) { bytes += TableLebBytes( ids.ref( 0x1e7683ef2ebc7684ull ) ) + 1 + 4; } // tick + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +LISTDEMO_TABLE_INLINE bool LogEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const LogEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.tick != 0 ) + { + w.putleb( ids.ref( 0x1e7683ef2ebc7684ull ) ); w.put8( 8 ); // tick + w.put32( uint32_t( value.tick ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +LISTDEMO_TABLE_INLINE bool LogEntryLoadBodyRetain( TableReader & r, LogEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + LogEntryReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x1e7683ef2ebc7684ull: // tick + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.tick = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.tick = decoded_v; break; } default: @@ -12008,6 +12585,86 @@ LISTDEMO_TABLE_INLINE bool LogEntryLoadBodyRetain( TableReader & r, LogEntry & v } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool LogEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, LogEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + LogEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x1e7683ef2ebc7684ull: // tick + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.tick = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.tick = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t SaveMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Save & value, TableRetain * retain, const TableRetainPath & path ) { @@ -12409,18 +13066,167 @@ inline bool SaveLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Sav } } -inline int64_t PointMeasureBodyRetain( TableRetainIds & ids, const Point & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.x != 0 ) { bytes += TableLebBytes( ids.ref( 0xaf63f54c86021707ull ) ) + 1 + 4; } // x - if ( value.y != 0 ) { bytes += TableLebBytes( ids.ref( 0xaf63f44c86021554ull ) ) + 1 + 4; } // y - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -LISTDEMO_TABLE_INLINE bool PointSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Point & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SaveLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Save & value, TableRetain * retain, const TableRetainPath & path ) { - if ( value.x != 0 ) + (void) nodes; (void) index_bits; + SaveReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xd24733aa574d4b09ull: // placements + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableListFill fill = TableListFillBegin( nodes, value.placements, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Placement * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + if ( !PlacementLoadMessageBodyRetain( r, vocabulary, report, index_bits, ( *slot ), retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { return false; } + } + TableListFillEnd( fill ); + } + break; + } + case 0x125073191daf5431ull: // log + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.log, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( *slot ), node_index_2, 0x5e781536ac58825full, report ); // *LogEntry + } + } + TableListFillEnd( fill ); + } + break; + } + case 0x01986b0b27400fb2ull: // scores + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 4 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.scores, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + int32_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + ( *slot ) = decoded_v_2; + } + } + TableListFillEnd( fill ); + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t PointMeasureBodyRetain( TableRetainIds & ids, const Point & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.x != 0 ) { bytes += TableLebBytes( ids.ref( 0xaf63f54c86021707ull ) ) + 1 + 4; } // x + if ( value.y != 0 ) { bytes += TableLebBytes( ids.ref( 0xaf63f44c86021554ull ) ) + 1 + 4; } // y + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +LISTDEMO_TABLE_INLINE bool PointSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Point & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.x != 0 ) { w.putleb( ids.ref( 0xaf63f54c86021707ull ) ); w.put8( 4 ); // x w.put32( uint32_t( value.x ) ); @@ -12527,6 +13333,148 @@ LISTDEMO_TABLE_INLINE bool PointLoadBodyRetain( TableReader & r, Point & value, } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool PointLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Point & value, TableRetain * retain, const TableRetainPath & path ) +{ + PointReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xaf63f54c86021707ull: // x + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.x = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.x = decoded_v; + } + break; + } + case 0xaf63f44c86021554ull: // y + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.y = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.y = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t MixedMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Mixed & value, TableRetain * retain, const TableRetainPath & path ) { @@ -13096,53 +14044,319 @@ inline bool MixedLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Mi } while ( 0 ); if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded } - TableListFillEnd( fill ); + TableListFillEnd( fill ); + } + else + { + // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped + // cannot fire on it. A count above the int32 storage cap is the + // fill's refusal, and it moves no counter. + TableListFill fill = TableListFillBegin( nodes, value.bounds, count ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + // elements are BOUNDED by the field body: a count the length cannot + // cover keeps the decoded prefix, flags malformed, and the parent + // continues at the next field + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < count; i++ ) + { + int32_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve + bool landed = false; + do + { + if ( !sub.has( 4 ) ) { r.report->malformed = true; break; } + int32_t decoded_v = int32_t( sub.get32( ) ); + if ( decoded_v < 0 ) { decoded_v = 0; r.report->clamped++; } + else if ( decoded_v > 100 ) { decoded_v = 100; r.report->clamped++; } + ( *slot ) = decoded_v; + landed = true; + } while ( 0 ); + if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + } + TableListFillEnd( fill ); + } + } + r.offset = body_end; // excess bytes and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool MixedLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Mixed & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + MixedReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xd90a4e7682f799c5ull: // grades + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 30 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.grades, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Grade * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t variant_ref_2 = 0; + if ( !r.get( variant_ref_2, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry variant_entry_2; + if ( variant_ref_2 == 0 ) { ( *slot ) = Grade::None; } // the zero reference is the enum's None + else if ( !TableMessageNameEntry( vocabulary, variant_ref_2, variant_entry_2 ) ) { report->malformed = true; return false; } + else if ( !TableEnumValue( variant_entry_2.id, ( *slot ) ) ) { ( *slot ) = Grade::None; report->unknown++; report->retain_lost++; } + } + } + TableListFillEnd( fill ); + } + break; + } + case 0x4af2ed8470862ea8ull: // perms + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 9 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.perms, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Perm * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + uint64_t decoded_v_2 = (uint64_t) decoded_wide_2; + ( *slot ) = decoded_v_2; + } + } + TableListFillEnd( fill ); + } + break; + } + case 0x732dfbcc9b0cf0bbull: // hits + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 15 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 2 ); + TableListFill fill = TableListFillBegin( nodes, value.hits, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Hit * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t arm_ref_2 = 0; + if ( !r.get( arm_ref_2, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm_2; + if ( arm_ref_2 == 0 ) { ( *slot ).type = HitType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref_2, arm_2 ) ) { report->malformed = true; return false; } + else + { + switch ( arm_2.id ) + { + case 0x73feab3544c345b1ull: // point + { + if ( arm_2.kind != 13 || arm_2.elem_kind != 0 ) + { + ( *slot ).type = HitType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( *slot ).type = HitType::Point; + if ( !PointLoadMessageBodyRetain( r, vocabulary, report, index_bits, ( *slot ).point, retain, TableRetainStepInto( path, 2, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0x7f6308be8ab37fc0ull: // damage + { + if ( arm_2.kind != 4 || arm_2.elem_kind != 0 ) + { + if ( TableKindWidens( arm_2.kind, 4 ) ) + { + ( *slot ).type = HitType::Damage; + memset( (void *) &( *slot ).damage, 0, sizeof( ( *slot ).damage ) ); // selection establishes the arm (§2.6) + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( *slot ).damage = decoded_v_3; + } + report->widened++; + break; + } + ( *slot ).type = HitType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + ( *slot ).type = HitType::Damage; + { + const int64_t width_3 = arm_2.value_bits; + uint64_t raw_3 = 0; + if ( width_3 < 0 || !r.get( raw_3, width_3 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_3 = (int64_t) raw_3; + if ( arm_2.packing == 1 ) { decoded_wide_3 = (int64_t) ( raw_3 + (uint64_t) arm_2.base_lo ); } + else if ( width_3 > 0 && width_3 < 64 ) + { + const uint64_t sign_3 = uint64_t(1) << ( width_3 - 1 ); + if ( ( raw_3 & sign_3 ) != 0 ) { decoded_wide_3 = (int64_t) ( raw_3 | ~( ( uint64_t(1) << width_3 ) - 1 ) ); } + } + if ( decoded_wide_3 < -2147483648ll ) { decoded_wide_3 = -2147483648ll; report->clamped++; } + if ( decoded_wide_3 > 2147483647ll ) { decoded_wide_3 = 2147483647ll; report->clamped++; } + int32_t decoded_v_3 = (int32_t) decoded_wide_3; + ( *slot ).damage = decoded_v_3; + } + break; + } + default: + ( *slot ).type = HitType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm_2 ) ) { report->malformed = true; return false; } + break; + } + } + } } - else + TableListFillEnd( fill ); + } + break; + } + case 0x52f60c4caef0b768ull: // bounds + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 4 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.bounds, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) { - // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped - // cannot fire on it. A count above the int32 storage cap is the - // fill's refusal, and it moves no counter. - TableListFill fill = TableListFillBegin( nodes, value.bounds, count ); - if ( fill.refused ) { nodes.refused = true; return false; } - if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } - // elements are BOUNDED by the field body: a count the length cannot - // cover keeps the decoded prefix, flags malformed, and the parent - // continues at the next field - TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); - for ( uint64_t i = 0; i < count; i++ ) + int32_t * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve { - int32_t * slot = TableListFillNext( fill ); - if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve - bool landed = false; - do + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) { - if ( !sub.has( 4 ) ) { r.report->malformed = true; break; } - int32_t decoded_v = int32_t( sub.get32( ) ); - if ( decoded_v < 0 ) { decoded_v = 0; r.report->clamped++; } - else if ( decoded_v > 100 ) { decoded_v = 100; r.report->clamped++; } - ( *slot ) = decoded_v; - landed = true; - } while ( 0 ); - if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < 0ll ) { decoded_wide_2 = 0ll; report->clamped++; } + if ( decoded_wide_2 > 100ll ) { decoded_wide_2 = 100ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + ( *slot ) = decoded_v_2; } - TableListFillEnd( fill ); } + TableListFillEnd( fill ); } - r.offset = body_end; // excess bytes and slack skip via the length - break; - } - case 0xffffffffffffffffull: - { - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } break; } default: + report->unknown++; { - r.report->unknown++; - if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } - break; + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); } + break; } } } @@ -13253,71 +14467,247 @@ inline const Save * SaveLoadRetain( uint8_t * region, int64_t region_bytes, cons // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Save ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = SaveNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + SaveNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + SaveNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Save ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + SaveLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// SaveNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool SaveNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + SaveNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !SaveNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x5e781536ac58825full: ok = LogEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(LogEntry *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // LogEntry + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// SaveLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool SaveLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Save * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !SaveMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = SaveNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + SaveNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !SaveMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Save ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x33f85f24c0f5f008ull; + Save * root = new ( region + used ) Save; // lifetime only: LoadMessageBody's first act is SaveReset + SaveReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Save ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Save ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = SaveNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - SaveNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - SaveNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !SaveNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Save ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - SaveLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return SaveLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// SaveLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool SaveLoadRetainMessages( const Save ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !SaveLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // SaveMeasureRetain and SaveSaveRetain: the pair, with the retained tail in @@ -13611,6 +15001,182 @@ inline const Mixed * MixedLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// MixedNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool MixedNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + MixedNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !MixedNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// MixedLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool MixedLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Mixed * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !MixedMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = MixedNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + MixedNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !MixedMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Mixed ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xbb86c6c96f598bb8ull; + Mixed * root = new ( region + used ) Mixed; // lifetime only: LoadMessageBody's first act is MixedReset + MixedReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Mixed ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !MixedNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return MixedLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// MixedLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool MixedLoadRetainMessages( const Mixed ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !MixedLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // MixedMeasureRetain and MixedSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/lists/SharedTable.h b/testdata/golden/tables/lists/SharedTable.h index 460636d46..2d96e5e3e 100644 --- a/testdata/golden/tables/lists/SharedTable.h +++ b/testdata/golden/tables/lists/SharedTable.h @@ -4278,6 +4278,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace listdemo #endif // LISTDEMO_SCHEMA_TABLE_RETAIN @@ -6013,10 +6445,12 @@ template inline bool TableNodeMessageSave( const Ctx & ctx, const inline int64_t PhotoMeasureBodyRetain( TableRetainIds & ids, const Photo & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool PhotoSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Photo & value, TableRetain * retain, const TableRetainPath & path ); LISTDEMO_TABLE_INLINE bool PhotoLoadBodyRetain( TableReader & r, Photo & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PhotoLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Photo & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t AlbumMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Album & value, TableRetain * retain, const TableRetainPath & path ); template inline bool AlbumSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Album & value, TableRetain * retain, const TableRetainPath & path ); template inline bool AlbumSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Album & value, TableRetain * retain, const TableRetainPath & path ); inline bool AlbumLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Album & value, TableRetain * retain, const TableRetainPath & path ); +inline bool AlbumLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Album & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t PhotoMeasureBody( TableIds & ids, const Photo & value ) { @@ -8248,6 +8682,124 @@ LISTDEMO_TABLE_INLINE bool PhotoLoadBodyRetain( TableReader & r, Photo & value, } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool PhotoLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Photo & value, TableRetain * retain, const TableRetainPath & path ) +{ + PhotoReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xdbdacd932fd1e9bfull: // width + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.width = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.width = decoded_v; + } + break; + } + case 0x17720bf67d347222ull: // height + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.height = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.height = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t AlbumMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Album & value, TableRetain * retain, const TableRetainPath & path ) { @@ -8469,6 +9021,99 @@ inline bool AlbumLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Al } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool AlbumLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Album & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + AlbumReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x40b1d94aff3ab130ull: // photos + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.photos, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( *slot ), node_index_2, 0xf1a78dd2508964c3ull, report ); // *Photo + } + } + TableListFillEnd( fill ); + } + break; + } + case 0xaa19a78e404dea20ull: // cover + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.cover, node_index, 0xf1a78dd2508964c3ull, report ); // *Photo + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // AlbumNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8642,6 +9287,182 @@ inline const Album * AlbumLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// AlbumNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool AlbumNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + AlbumNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !AlbumNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0xf1a78dd2508964c3ull: ok = PhotoLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Photo *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Photo + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// AlbumLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool AlbumLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Album * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !AlbumMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = AlbumNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + AlbumNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !AlbumMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Album ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xd858c2cb7f1514ccull; + Album * root = new ( region + used ) Album; // lifetime only: LoadMessageBody's first act is AlbumReset + AlbumReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Album ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !AlbumNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return AlbumLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// AlbumLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool AlbumLoadRetainMessages( const Album ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !AlbumLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // AlbumMeasureRetain and AlbumSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/CellsTable.h b/testdata/golden/tables/maps/CellsTable.h index 161361a45..e9094640a 100644 --- a/testdata/golden/tables/maps/CellsTable.h +++ b/testdata/golden/tables/maps/CellsTable.h @@ -4399,6 +4399,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6239,10 +6671,12 @@ inline CellsRowsEntryKeyRead CellsRowsEntryReadKey( const uint8_t * body, int64_ inline int64_t CellsRowsEntryMeasureBodyRetain( TableRetainIds & ids, const CellsRowsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool CellsRowsEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const CellsRowsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool CellsRowsEntryLoadBodyRetain( TableReader & r, CellsRowsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CellsRowsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, CellsRowsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t CellsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Cells & value, TableRetain * retain, const TableRetainPath & path ); template inline bool CellsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Cells & value, TableRetain * retain, const TableRetainPath & path ); template inline bool CellsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Cells & value, TableRetain * retain, const TableRetainPath & path ); inline bool CellsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Cells & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CellsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Cells & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t CellsRowsEntryMeasureBody( TableIds & ids, const CellsRowsEntry & value ) { @@ -8386,56 +8820,219 @@ MAPDEMO_TABLE_INLINE bool CellsRowsEntryLoadBodyRetain( TableReader & r, CellsRo } } -template -inline int64_t CellsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Cells & value, TableRetain * retain, const TableRetainPath & path ) -{ - (void) ctx; (void) numbering; - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // rows: a kind 14 array of kind 13 elements, ASCENDING (§2.8) - TableMapCursor order_rows = TableMapOrder( ctx, value.rows ); - if ( !order_rows.ok ) { return -1; } // the sort could not run - if ( order_rows.count > 0 ) - { - const uint64_t ref_rows = ids.ref( 0xa3a7061ff10a8138ull ); - int64_t body_rows = 1 + TableLebBytes( (uint64_t) order_rows.count ); // the element kind byte and the count - for ( int32_t i = 0; i < order_rows.count; i++ ) - { - const int64_t elem_rows = CellsRowsEntryMeasureBodyRetain( ids, *order_rows[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_rows < 0 ) { TableMapRelease( order_rows ); return -1; } - body_rows += TableLebBytes( (uint64_t) ( elem_rows ) ) + ( elem_rows ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key - } - bytes += TableLebBytes( ref_rows ) + 1 + TableLebBytes( (uint64_t) ( body_rows ) ) + ( body_rows ); - } - TableMapRelease( order_rows ); - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool CellsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Cells & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool CellsRowsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, CellsRowsEntry & value, TableRetain * retain, const TableRetainPath & path ) { - (void) ctx; (void) numbering; + CellsRowsEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - TableMapCursor order_rows = TableMapOrder( ctx, value.rows ); // rows - if ( !order_rows.ok ) { return false; } - if ( order_rows.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - const uint64_t ref_rows = ids.ref( 0xa3a7061ff10a8138ull ); - int64_t body_rows = 1 + TableLebBytes( (uint64_t) order_rows.count ); - for ( int32_t i = 0; i < order_rows.count; i++ ) + case 0x3dc94a19365b10ecull: // key { - const int64_t elem_rows = CellsRowsEntryMeasureBodyRetain( ids, *order_rows[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_rows < 0 ) { TableMapRelease( order_rows ); return false; } - body_rows += TableLebBytes( (uint64_t) ( elem_rows ) ) + ( elem_rows ); + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); + value.key[kept] = 0; + r.offset += (int64_t) n * 8; + value.key_length = kept; + } + break; } - w.putleb( ref_rows ); w.put8( 14 ); w.putleb( (uint64_t) body_rows ); - w.put8( 13 ); w.putleb( (uint64_t) order_rows.count ); - for ( int32_t i = 0; i < order_rows.count; i++ ) + case 0x7ce4fd9430e80ceaull: // value { - const int64_t elem_len_rows = CellsRowsEntryMeasureBodyRetain( ids, *order_rows[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 4 ) + { + if ( entry.kind == 14 && TableKindWidens( entry.elem_kind, 4 ) ) + { + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 3 ) { kept = 3; report->clamped++; } else { kept = (int32_t) n; } + const int64_t surplus_bits = entry.elem_value_bits; + uint64_t walk = n; + if ( surplus_bits >= 0 && walk > (uint64_t) 3 ) { walk = (uint64_t) 3; } // the surplus is arithmetic + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + int32_t scratch = 0; + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + scratch = decoded_v_2; + } + if ( in_bounds ) { value.value[i] = scratch; } + } + if ( walk < n && !TableMessageSkipRun( r, n - walk, surplus_bits ) ) { report->malformed = true; return false; } + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 3 ) { kept = 3; report->clamped++; } else { kept = (int32_t) n; } + const int64_t surplus_bits = entry.elem_value_bits; + uint64_t walk = n; + if ( surplus_bits >= 0 && walk > (uint64_t) 3 ) { walk = (uint64_t) 3; } // the surplus is arithmetic + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + int32_t scratch = 0; + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + scratch = decoded_v_2; + } + if ( in_bounds ) { value.value[i] = scratch; } + } + if ( walk < n && !TableMessageSkipRun( r, n - walk, surplus_bits ) ) { report->malformed = true; return false; } + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t CellsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Cells & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // rows: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_rows = TableMapOrder( ctx, value.rows ); + if ( !order_rows.ok ) { return -1; } // the sort could not run + if ( order_rows.count > 0 ) + { + const uint64_t ref_rows = ids.ref( 0xa3a7061ff10a8138ull ); + int64_t body_rows = 1 + TableLebBytes( (uint64_t) order_rows.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_rows.count; i++ ) + { + const int64_t elem_rows = CellsRowsEntryMeasureBodyRetain( ids, *order_rows[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_rows < 0 ) { TableMapRelease( order_rows ); return -1; } + body_rows += TableLebBytes( (uint64_t) ( elem_rows ) ) + ( elem_rows ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_rows ) + 1 + TableLebBytes( (uint64_t) ( body_rows ) ) + ( body_rows ); + } + TableMapRelease( order_rows ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool CellsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Cells & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + { + TableMapCursor order_rows = TableMapOrder( ctx, value.rows ); // rows + if ( !order_rows.ok ) { return false; } + if ( order_rows.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_rows = ids.ref( 0xa3a7061ff10a8138ull ); + int64_t body_rows = 1 + TableLebBytes( (uint64_t) order_rows.count ); + for ( int32_t i = 0; i < order_rows.count; i++ ) + { + const int64_t elem_rows = CellsRowsEntryMeasureBodyRetain( ids, *order_rows[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_rows < 0 ) { TableMapRelease( order_rows ); return false; } + body_rows += TableLebBytes( (uint64_t) ( elem_rows ) ) + ( elem_rows ); + } + w.putleb( ref_rows ); w.put8( 14 ); w.putleb( (uint64_t) body_rows ); + w.put8( 13 ); w.putleb( (uint64_t) order_rows.count ); + for ( int32_t i = 0; i < order_rows.count; i++ ) + { + const int64_t elem_len_rows = CellsRowsEntryMeasureBodyRetain( ids, *order_rows[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); if ( elem_len_rows < 0 ) { TableMapRelease( order_rows ); return false; } w.putleb( (uint64_t) elem_len_rows ); if ( !CellsRowsEntrySaveBodyRetain( w, ids, *order_rows[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_rows ); return false; } @@ -8622,6 +9219,164 @@ inline bool CellsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Ce } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool CellsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Cells & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + CellsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xa3a7061ff10a8138ull: // rows + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.rows, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + const char * last_key = NULL; int32_t last_length = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const CellsRowsEntryMessageKeyRead read = CellsRowsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( last_key, last_length, read.key, read.length ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + CellsRowsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !CellsRowsEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; last_length = read.length; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // CellsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8795,6 +9550,182 @@ inline const Cells * CellsLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// CellsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool CellsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + CellsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !CellsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// CellsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool CellsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Cells * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !CellsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = CellsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + CellsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !CellsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Cells ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x1f781dc01a2b5152ull; + Cells * root = new ( region + used ) Cells; // lifetime only: LoadMessageBody's first act is CellsReset + CellsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Cells ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !CellsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return CellsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// CellsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool CellsLoadRetainMessages( const Cells ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !CellsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // CellsMeasureRetain and CellsSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/ChunksTable.h b/testdata/golden/tables/maps/ChunksTable.h index b6de29938..36df48479 100644 --- a/testdata/golden/tables/maps/ChunksTable.h +++ b/testdata/golden/tables/maps/ChunksTable.h @@ -4399,6 +4399,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6241,10 +6673,12 @@ template inline int64_t ChunksBlobsEntryMeasureBodyRetain( const template inline bool ChunksBlobsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const ChunksBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ChunksBlobsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const ChunksBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ); inline bool ChunksBlobsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, ChunksBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ChunksBlobsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, ChunksBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t ChunksMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ChunksSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ChunksSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ); inline bool ChunksLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Chunks & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ChunksLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Chunks & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t ChunksBlobsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const ChunksBlobsEntry & value ) @@ -8439,92 +8873,203 @@ inline bool ChunksBlobsEntryLoadBodyRetain( TableReader & r, const TableNodeMap } } -template -inline int64_t ChunksMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // blobs: a kind 14 array of kind 13 elements, ASCENDING (§2.8) - TableMapCursor order_blobs = TableMapOrder( ctx, value.blobs ); - if ( !order_blobs.ok ) { return -1; } // the sort could not run - if ( order_blobs.count > 0 ) - { - const uint64_t ref_blobs = ids.ref( 0x14e2eaab9cde925bull ); - int64_t body_blobs = 1 + TableLebBytes( (uint64_t) order_blobs.count ); // the element kind byte and the count - for ( int32_t i = 0; i < order_blobs.count; i++ ) - { - const int64_t elem_blobs = ChunksBlobsEntryMeasureBodyRetain( ctx, numbering, ids, *order_blobs[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_blobs < 0 ) { TableMapRelease( order_blobs ); return -1; } - body_blobs += TableLebBytes( (uint64_t) ( elem_blobs ) ) + ( elem_blobs ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key - } - bytes += TableLebBytes( ref_blobs ) + 1 + TableLebBytes( (uint64_t) ( body_blobs ) ) + ( body_blobs ); - } - TableMapRelease( order_blobs ); - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool ChunksSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ) -{ - { - TableMapCursor order_blobs = TableMapOrder( ctx, value.blobs ); // blobs - if ( !order_blobs.ok ) { return false; } - if ( order_blobs.count > 0 ) // an EMPTY map elides, the by-value rule (§3) - { - const uint64_t ref_blobs = ids.ref( 0x14e2eaab9cde925bull ); - int64_t body_blobs = 1 + TableLebBytes( (uint64_t) order_blobs.count ); - for ( int32_t i = 0; i < order_blobs.count; i++ ) - { - const int64_t elem_blobs = ChunksBlobsEntryMeasureBodyRetain( ctx, numbering, ids, *order_blobs[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_blobs < 0 ) { TableMapRelease( order_blobs ); return false; } - body_blobs += TableLebBytes( (uint64_t) ( elem_blobs ) ) + ( elem_blobs ); - } - w.putleb( ref_blobs ); w.put8( 14 ); w.putleb( (uint64_t) body_blobs ); - w.put8( 13 ); w.putleb( (uint64_t) order_blobs.count ); - for ( int32_t i = 0; i < order_blobs.count; i++ ) - { - const int64_t elem_len_blobs = ChunksBlobsEntryMeasureBodyRetain( ctx, numbering, ids, *order_blobs[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_len_blobs < 0 ) { TableMapRelease( order_blobs ); return false; } - w.putleb( (uint64_t) elem_len_blobs ); - if ( !ChunksBlobsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_blobs[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_blobs ); return false; } - } - } - TableMapRelease( order_blobs ); - } - if ( value.after != 0 ) - { - w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after - w.put32( uint32_t( value.after ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - return !w.overflow; -} - -template -inline bool ChunksSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( !ChunksSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -inline bool ChunksLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Chunks & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ChunksBlobsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, ChunksBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ) { - ChunksReset( value ); // prefill declared defaults in place, then overlay + (void) nodes; (void) index_bits; + ChunksBlobsEntryReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.value, node_index, kTableBytesTypeId, report ); // *bytes + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t ChunksMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // blobs: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_blobs = TableMapOrder( ctx, value.blobs ); + if ( !order_blobs.ok ) { return -1; } // the sort could not run + if ( order_blobs.count > 0 ) + { + const uint64_t ref_blobs = ids.ref( 0x14e2eaab9cde925bull ); + int64_t body_blobs = 1 + TableLebBytes( (uint64_t) order_blobs.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_blobs.count; i++ ) + { + const int64_t elem_blobs = ChunksBlobsEntryMeasureBodyRetain( ctx, numbering, ids, *order_blobs[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_blobs < 0 ) { TableMapRelease( order_blobs ); return -1; } + body_blobs += TableLebBytes( (uint64_t) ( elem_blobs ) ) + ( elem_blobs ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_blobs ) + 1 + TableLebBytes( (uint64_t) ( body_blobs ) ) + ( body_blobs ); + } + TableMapRelease( order_blobs ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool ChunksSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_blobs = TableMapOrder( ctx, value.blobs ); // blobs + if ( !order_blobs.ok ) { return false; } + if ( order_blobs.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_blobs = ids.ref( 0x14e2eaab9cde925bull ); + int64_t body_blobs = 1 + TableLebBytes( (uint64_t) order_blobs.count ); + for ( int32_t i = 0; i < order_blobs.count; i++ ) + { + const int64_t elem_blobs = ChunksBlobsEntryMeasureBodyRetain( ctx, numbering, ids, *order_blobs[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_blobs < 0 ) { TableMapRelease( order_blobs ); return false; } + body_blobs += TableLebBytes( (uint64_t) ( elem_blobs ) ) + ( elem_blobs ); + } + w.putleb( ref_blobs ); w.put8( 14 ); w.putleb( (uint64_t) body_blobs ); + w.put8( 13 ); w.putleb( (uint64_t) order_blobs.count ); + for ( int32_t i = 0; i < order_blobs.count; i++ ) + { + const int64_t elem_len_blobs = ChunksBlobsEntryMeasureBodyRetain( ctx, numbering, ids, *order_blobs[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_blobs < 0 ) { TableMapRelease( order_blobs ); return false; } + w.putleb( (uint64_t) elem_len_blobs ); + if ( !ChunksBlobsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_blobs[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_blobs ); return false; } + } + } + TableMapRelease( order_blobs ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool ChunksSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chunks & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !ChunksSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool ChunksLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Chunks & value, TableRetain * retain, const TableRetainPath & path ) +{ + ChunksReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count const uint64_t field_id = r.ids->at( field_ref ); if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } uint8_t kind = r.get8(); @@ -8672,6 +9217,164 @@ inline bool ChunksLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, C } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ChunksLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Chunks & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + ChunksReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x14e2eaab9cde925bull: // blobs + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.blobs, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + int32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const ChunksBlobsEntryMessageKeyRead read = ChunksBlobsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (int64_t) last_key, (int64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + ChunksBlobsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !ChunksBlobsEntryLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // ChunksNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8846,6 +9549,182 @@ inline const Chunks * ChunksLoadRetain( uint8_t * region, int64_t region_bytes, return root; } +// ChunksNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool ChunksNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + ChunksNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !ChunksNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// ChunksLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool ChunksLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Chunks * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !ChunksMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = ChunksNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + ChunksNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !ChunksMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Chunks ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x816c207ba6213983ull; + Chunks * root = new ( region + used ) Chunks; // lifetime only: LoadMessageBody's first act is ChunksReset + ChunksReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Chunks ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !ChunksNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return ChunksLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// ChunksLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool ChunksLoadRetainMessages( const Chunks ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !ChunksLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // ChunksMeasureRetain and ChunksSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/CrewsTable.h b/testdata/golden/tables/maps/CrewsTable.h index 0acb2ac69..67b4e681e 100644 --- a/testdata/golden/tables/maps/CrewsTable.h +++ b/testdata/golden/tables/maps/CrewsTable.h @@ -4400,6 +4400,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6240,10 +6672,12 @@ template inline int64_t CrewsMembersEntryMeasureBodyRetain( const template inline bool CrewsMembersEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool CrewsMembersEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); inline bool CrewsMembersEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CrewsMembersEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t CrewsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); template inline bool CrewsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); template inline bool CrewsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); inline bool CrewsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Crews & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CrewsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Crews & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t CrewsMembersEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const CrewsMembersEntry & value ) @@ -8577,92 +9011,211 @@ inline bool CrewsMembersEntryLoadBodyRetain( TableReader & r, const TableNodeMap } } -template -inline int64_t CrewsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // members: a kind 14 array of kind 13 elements, ASCENDING (§2.8) - TableMapCursor order_members = TableMapOrder( ctx, value.members ); - if ( !order_members.ok ) { return -1; } // the sort could not run - if ( order_members.count > 0 ) - { - const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); - int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); // the element kind byte and the count - for ( int32_t i = 0; i < order_members.count; i++ ) - { - const int64_t elem_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_members < 0 ) { TableMapRelease( order_members ); return -1; } - body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key - } - bytes += TableLebBytes( ref_members ) + 1 + TableLebBytes( (uint64_t) ( body_members ) ) + ( body_members ); - } - TableMapRelease( order_members ); - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool CrewsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) -{ - { - TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members - if ( !order_members.ok ) { return false; } - if ( order_members.count > 0 ) // an EMPTY map elides, the by-value rule (§3) - { - const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); - int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); - for ( int32_t i = 0; i < order_members.count; i++ ) - { - const int64_t elem_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_members < 0 ) { TableMapRelease( order_members ); return false; } - body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); - } - w.putleb( ref_members ); w.put8( 14 ); w.putleb( (uint64_t) body_members ); - w.put8( 13 ); w.putleb( (uint64_t) order_members.count ); - for ( int32_t i = 0; i < order_members.count; i++ ) - { - const int64_t elem_len_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_len_members < 0 ) { TableMapRelease( order_members ); return false; } - w.putleb( (uint64_t) elem_len_members ); - if ( !CrewsMembersEntrySaveBodyRetain( ctx, numbering, w, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_members ); return false; } - } - } - TableMapRelease( order_members ); - } - if ( value.after != 0 ) - { - w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after - w.put32( uint32_t( value.after ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - return !w.overflow; -} - -template -inline bool CrewsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( !CrewsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -inline bool CrewsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Crews & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool CrewsMembersEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) { - CrewsReset( value ); // prefill declared defaults in place, then overlay + (void) nodes; (void) index_bits; + CrewsMembersEntryReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.value[i] : scratch ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + value.value_count = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t CrewsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // members: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_members = TableMapOrder( ctx, value.members ); + if ( !order_members.ok ) { return -1; } // the sort could not run + if ( order_members.count > 0 ) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return -1; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_members ) + 1 + TableLebBytes( (uint64_t) ( body_members ) ) + ( body_members ); + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool CrewsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return false; } + if ( order_members.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return false; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); + } + w.putleb( ref_members ); w.put8( 14 ); w.putleb( (uint64_t) body_members ); + w.put8( 13 ); w.putleb( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_len_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_members < 0 ) { TableMapRelease( order_members ); return false; } + w.putleb( (uint64_t) elem_len_members ); + if ( !CrewsMembersEntrySaveBodyRetain( ctx, numbering, w, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_members ); return false; } + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool CrewsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !CrewsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + CrewsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count const uint64_t field_id = r.ids->at( field_ref ); if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } uint8_t kind = r.get8(); @@ -8810,6 +9363,164 @@ inline bool CrewsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Cr } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool CrewsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + CrewsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x79d594675e391090ull: // members + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.members, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const CrewsMembersEntryMessageKeyRead read = CrewsMembersEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + CrewsMembersEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !CrewsMembersEntryLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // CrewsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8983,6 +9694,182 @@ inline const Crews * CrewsLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// CrewsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool CrewsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + CrewsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !CrewsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// CrewsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool CrewsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Crews * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !CrewsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = CrewsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + CrewsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !CrewsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xc85b940060088651ull; + Crews * root = new ( region + used ) Crews; // lifetime only: LoadMessageBody's first act is CrewsReset + CrewsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Crews ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !CrewsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return CrewsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// CrewsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool CrewsLoadRetainMessages( const Crews ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !CrewsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // CrewsMeasureRetain and CrewsSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/DepthTable.h b/testdata/golden/tables/maps/DepthTable.h index 7cf956f68..c597bbaf1 100644 --- a/testdata/golden/tables/maps/DepthTable.h +++ b/testdata/golden/tables/maps/DepthTable.h @@ -4400,6 +4400,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6364,14 +6796,17 @@ inline SquadRosterEntryKeyRead SquadRosterEntryReadKey( const uint8_t * body, in inline int64_t SquadRosterEntryMeasureBodyRetain( TableRetainIds & ids, const SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool SquadRosterEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool SquadRosterEntryLoadBodyRetain( TableReader & r, SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SquadRosterEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t SquadMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Squad & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SquadSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Squad & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SquadSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Squad & value, TableRetain * retain, const TableRetainPath & path ); inline bool SquadLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Squad & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SquadLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Squad & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t DepthMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Depth & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DepthSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Depth & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DepthSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Depth & value, TableRetain * retain, const TableRetainPath & path ); inline bool DepthLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Depth & value, TableRetain * retain, const TableRetainPath & path ); +inline bool DepthLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Depth & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t SquadRosterEntryMeasureBody( TableIds & ids, const SquadRosterEntry & value ) { @@ -10367,6 +10802,86 @@ MAPDEMO_TABLE_INLINE bool SquadRosterEntryLoadBodyRetain( TableReader & r, Squad } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SquadRosterEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, SquadRosterEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + SquadRosterEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 6 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 255ull ) { decoded_wide = (int64_t) 255ull; report->clamped++; } + uint8_t decoded_v = (uint8_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t SquadMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Squad & value, TableRetain * retain, const TableRetainPath & path ) { @@ -10571,16 +11086,124 @@ inline bool SquadLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Sq } } -template -inline int64_t DepthMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Depth & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SquadLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Squad & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body + (void) nodes; (void) index_bits; + SquadReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - const int32_t mark_one = ids.count; - const uint64_t ref_one = ids.ref( 0x1a08aa1921ca5cafull ); - const int64_t body_one = SquadMeasureBodyRetain( ctx, numbering, ids, value.one, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ); - if ( body_one < 0 ) { return -1; } - if ( body_one > 1 ) { bytes += TableLebBytes( ref_one ) + 1 + TableLebBytes( (uint64_t) ( body_one ) ) + ( body_one ); } // one + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x1c84390d304f4f42ull: // roster + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.roster, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint8_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const SquadRosterEntryMessageKeyRead read = SquadRosterEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + SquadRosterEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !SquadRosterEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t DepthMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Depth & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + const int32_t mark_one = ids.count; + const uint64_t ref_one = ids.ref( 0x1a08aa1921ca5cafull ); + const int64_t body_one = SquadMeasureBodyRetain( ctx, numbering, ids, value.one, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ); + if ( body_one < 0 ) { return -1; } + if ( body_one > 1 ) { bytes += TableLebBytes( ref_one ) + 1 + TableLebBytes( (uint64_t) ( body_one ) ) + ( body_one ); } // one else { ids.truncate( mark_one ); } // an all-default nested table elides, and costs no entry } if ( value.many_count < 0 || value.many_count > 3 ) { return -1; } // storage invariant @@ -11067,177 +11690,621 @@ inline bool DepthLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, De } } -// SquadNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void SquadNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) -{ - // the node's own EXTENT, where its lists' and maps' arrays are carved - // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). - // The tool's path carries a worker instead: there the arrays are the - // arena's. - TableExtentCarve carve; - carve.worker = nodes.worker; - if ( carve.worker == NULL ) - { - TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse - const int64_t storage = SquadNodeStorage( type_id, r.size, reason ); - const int64_t record = storage > 0 ? SquadNodeRecordBytes( type_id ) : 0; - carve.at = at + record; - carve.left = storage > record ? storage - record : 0; - } - nodes.carve = &carve; - (void) nodes; // every node this root can name is a FIXED table - (void) retain; (void) node; - switch ( type_id ) - { - default: break; - } - nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done -} - -// SquadLoadRetain: decode the tolerant wire into the caller's exact-sized region and -// return the root. LOAD IS A SCAN, and that is the whole of its bound: it -// follows no reference, so there is no depth cap, no visited set and no -// ordering rule on the indices. Partial results are kept, as everywhere on -// this wire — the report says what happened. NULL means the CALLER's buffer -// was wrong. -// UNDER RETENTION it also fills the caller's two stores with the fields -// this build cannot name, and the report carries what it could not keep -// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the -// reader's data is exactly what it would have been with retention off. -inline const Squad * SquadLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool DepthLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Depth & value, TableRetain * retain, const TableRetainPath & path ) { - TableReport ignored; - TableReport * out = report != NULL ? report : &ignored; - // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: - // a file that is both a newer form and damaged is a REFUSAL and never - // damage (docs/SPEC-TABLES.md §3). - TableIdTable ids_table; - int64_t body_bytes = 0; - const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); - if ( verdict != TableOpenOk ) - { - if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } - return NULL; - } - if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) - { - out->malformed = true; // a byte no field claims, before the table (§3) - return NULL; - } - const uint8_t * const wire = wire_file + 1; - const int64_t wire_bytes = body_bytes; - if ( region == NULL || region_bytes < (int64_t) sizeof( Squad ) ) { out->malformed = true; return NULL; } - if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } - memset( region, 0, (size_t) region_bytes ); - uint64_t type_id = 0; - const uint8_t * body = NULL; - int64_t length = 0; - - // the record count and the data bytes, from the FRAMING alone - TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed - int64_t root_extent = 0; - if ( !SquadWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } - int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Squad ) ) + root_extent ); - int64_t records = 0; - { - TableReport counting; - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); - while ( TableNodeScanNext( scan, type_id, body, length ) ) - { - records++; - int64_t storage = SquadNodeStorage( type_id, length, reason ); - if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } - if ( storage > 0 ) { data += storage; } - } - } - int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); - if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } - - TableNodeMap nodes; - nodes.base = region; - nodes.entries = (const TableNodeDirEntry *) ( region + data ); - nodes.count = records + 1; - TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); - directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) - directory[0].type_id = 0xec07a2f760550a91ull; - Squad * root = new ( region ) Squad; // lifetime only: LoadBody's first act is SquadReset - SquadReset( *root ); - - // LoadRetain RESETS BOTH STORES and writes into neither id list: a - // retained record carries its field's identity in the record itself, - // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer - // belongs to this region from here on. - TableRetainReset( retain, nodes, region ); - - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + (void) nodes; (void) index_bits; + DepthReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Squad ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - int64_t storage = SquadNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) + case 0x1a08aa1921ca5cafull: // one { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 0 ); + if ( !SquadLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, value.one, retain, TableRetainStepInto( path, 0, (uint32_t) ( 0 ) ) ) ) { return false; } + break; } - else + case 0x1f6459a2cea1fc02ull: // many { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - SquadNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 3 ) { kept = 3; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + Squad scratch; + SquadReset( scratch ); + if ( !SquadLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( in_bounds ? value.many[i] : scratch ), ( in_bounds ? retain : NULL ), TableRetainStepInto( path, 1, (uint32_t) ( i ) ) ) ) { return false; } + } + value.many_count = kept; + } + break; + } + case 0x70551ff29550f15dull: // keyed + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 16 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { report->malformed = true; return false; } + for ( uint64_t p = 0; p < n; p++ ) + { + uint64_t key_ref = 0; + if ( !r.get( key_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry key_entry; + if ( !TableMessageNameEntry( vocabulary, key_ref, key_entry ) ) { report->malformed = true; return false; } + Slot key = Slot::None; + const bool named = TableEnumValue( key_entry.id, key ); + if ( !named ) { report->unknown++; report->retain_lost++; } + const int32_t slot = named ? (int32_t) key - 1 : -1; + const bool in_bounds = slot >= 0 && slot < 2; + Squad scratch; + SquadReset( scratch ); + if ( !SquadLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( in_bounds ? value.keyed.slots[slot] : scratch ), ( in_bounds ? retain : NULL ), TableRetainStepInto( path, 2, (uint32_t) ( slot ) ) ) ) { return false; } + } + } + break; + } + case 0xe756c0190570ccb5ull: // arm + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 15 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 3 ); + { + uint64_t arm_ref = 0; + if ( !r.get( arm_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm; + if ( arm_ref == 0 ) { value.arm.type = ForceType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref, arm ) ) { report->malformed = true; return false; } + else + { + switch ( arm.id ) + { + case 0xd5c2bb95d63e6331ull: // squad + { + if ( arm.kind != 13 || arm.elem_kind != 0 ) + { + value.arm.type = ForceType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.arm.type = ForceType::Squad; + if ( !SquadLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, value.arm.squad, retain, TableRetainStepInto( path, 3, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0xfd4d194e1652b207ull: // plain + { + if ( arm.kind != 4 || arm.elem_kind != 0 ) + { + if ( TableKindWidens( arm.kind, 4 ) ) + { + value.arm.type = ForceType::Plain; + memset( (void *) &value.arm.plain, 0, sizeof( value.arm.plain ) ); // selection establishes the arm (§2.6) + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.arm.plain = decoded_v_2; + } + report->widened++; + break; + } + value.arm.type = ForceType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.arm.type = ForceType::Plain; + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + value.arm.plain = decoded_v_2; + } + break; + } + default: + value.arm.type = ForceType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + } + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// SquadNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void SquadNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = SquadNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? SquadNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + (void) retain; (void) node; + switch ( type_id ) + { + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// SquadLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Squad * SquadLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Squad ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !SquadWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Squad ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = SquadNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xec07a2f760550a91ull; + Squad * root = new ( region ) Squad; // lifetime only: LoadBody's first act is SquadReset + SquadReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Squad ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = SquadNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + SquadNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + SquadNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Squad ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + SquadLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// SquadNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool SquadNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + SquadNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !SquadNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// SquadLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool SquadLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Squad * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !SquadMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = SquadNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + SquadNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !SquadMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Squad ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xec07a2f760550a91ull; + Squad * root = new ( region + used ) Squad; // lifetime only: LoadMessageBody's first act is SquadReset + SquadReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Squad ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - SquadNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !SquadNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Squad ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - SquadLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return SquadLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// SquadLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool SquadLoadRetainMessages( const Squad ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !SquadLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // SquadMeasureRetain and SquadSaveRetain: the pair, with the retained tail in @@ -11528,6 +12595,182 @@ inline const Depth * DepthLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// DepthNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool DepthNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + DepthNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !DepthNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// DepthLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool DepthLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Depth * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !DepthMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = DepthNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + DepthNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !DepthMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Depth ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x3231d0dc6fe30d4aull; + Depth * root = new ( region + used ) Depth; // lifetime only: LoadMessageBody's first act is DepthReset + DepthReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Depth ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !DepthNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return DepthLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// DepthLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool DepthLoadRetainMessages( const Depth ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !DepthLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // DepthMeasureRetain and DepthSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/DocsTable.h b/testdata/golden/tables/maps/DocsTable.h index da13c5c5d..e3e144b53 100644 --- a/testdata/golden/tables/maps/DocsTable.h +++ b/testdata/golden/tables/maps/DocsTable.h @@ -4399,6 +4399,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6248,10 +6680,12 @@ template inline int64_t DocsPagesEntryMeasureBodyRetain( const Ct template inline bool DocsPagesEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const DocsPagesEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DocsPagesEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const DocsPagesEntry & value, TableRetain * retain, const TableRetainPath & path ); inline bool DocsPagesEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, DocsPagesEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool DocsPagesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, DocsPagesEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t DocsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Docs & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DocsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Docs & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DocsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Docs & value, TableRetain * retain, const TableRetainPath & path ); inline bool DocsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Docs & value, TableRetain * retain, const TableRetainPath & path ); +inline bool DocsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Docs & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t DocsPagesEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const DocsPagesEntry & value ) @@ -8447,6 +8881,92 @@ inline bool DocsPagesEntryLoadBodyRetain( TableReader & r, const TableNodeMap & } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool DocsPagesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, DocsPagesEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + DocsPagesEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); + value.key[kept] = 0; + r.offset += (int64_t) n * 8; + value.key_length = kept; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.value, node_index, kTableStringTypeId, report ); // *string + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t DocsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Docs & value, TableRetain * retain, const TableRetainPath & path ) { @@ -8680,14 +9200,172 @@ inline bool DocsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Doc } } -// DocsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void DocsNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool DocsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Docs & value, TableRetain * retain, const TableRetainPath & path ) { - // the node's own EXTENT, where its lists' and maps' arrays are carved + (void) nodes; (void) index_bits; + DocsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xab01daa76a48769full: // pages + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.pages, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + const char * last_key = NULL; int32_t last_length = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const DocsPagesEntryMessageKeyRead read = DocsPagesEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( last_key, last_length, read.key, read.length ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + DocsPagesEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !DocsPagesEntryLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; last_length = read.length; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// DocsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void DocsNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). // The tool's path carries a worker instead: there the arrays are the // arena's. @@ -8862,6 +9540,183 @@ inline const Docs * DocsLoadRetain( uint8_t * region, int64_t region_bytes, cons return root; } +// DocsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool DocsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + DocsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !DocsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// DocsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool DocsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Docs * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !DocsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = DocsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + DocsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !DocsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Docs ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x036ffe7360826852ull; + Docs * root = new ( region + used ) Docs; // lifetime only: LoadMessageBody's first act is DocsReset + DocsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Docs ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( type_id == kTableStringTypeId && !TableUtf8Valid( r.buffer + r.offset / 8, length ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !DocsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return DocsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// DocsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool DocsLoadRetainMessages( const Docs ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !DocsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // DocsMeasureRetain and DocsSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/FleetTable.h b/testdata/golden/tables/maps/FleetTable.h index 67ee3f150..43766c9e3 100644 --- a/testdata/golden/tables/maps/FleetTable.h +++ b/testdata/golden/tables/maps/FleetTable.h @@ -4399,6 +4399,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6992,30 +7424,38 @@ inline FleetTiersEntryKeyRead FleetTiersEntryReadKey( const uint8_t * body, int6 inline int64_t ShipConfigMeasureBodyRetain( TableRetainIds & ids, const ShipConfig & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool ShipConfigSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const ShipConfig & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool ShipConfigLoadBodyRetain( TableReader & r, ShipConfig & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ShipConfigLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, ShipConfig & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t ItemMeasureBodyRetain( TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool ItemSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool ItemLoadBodyRetain( TableReader & r, Item & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ItemLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Item & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t FleetShipsEntryMeasureBodyRetain( TableRetainIds & ids, const FleetShipsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool FleetShipsEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const FleetShipsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool FleetShipsEntryLoadBodyRetain( TableReader & r, FleetShipsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool FleetShipsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, FleetShipsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t FleetByIdEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const FleetByIdEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FleetByIdEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const FleetByIdEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FleetByIdEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const FleetByIdEntry & value, TableRetain * retain, const TableRetainPath & path ); inline bool FleetByIdEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, FleetByIdEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool FleetByIdEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, FleetByIdEntry & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t FleetLoadoutsEntryValueEntryMeasureBodyRetain( TableRetainIds & ids, const FleetLoadoutsEntryValueEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool FleetLoadoutsEntryValueEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const FleetLoadoutsEntryValueEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool FleetLoadoutsEntryValueEntryLoadBodyRetain( TableReader & r, FleetLoadoutsEntryValueEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool FleetLoadoutsEntryValueEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, FleetLoadoutsEntryValueEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t FleetLoadoutsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const FleetLoadoutsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FleetLoadoutsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const FleetLoadoutsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FleetLoadoutsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const FleetLoadoutsEntry & value, TableRetain * retain, const TableRetainPath & path ); inline bool FleetLoadoutsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, FleetLoadoutsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool FleetLoadoutsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, FleetLoadoutsEntry & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t FleetTiersEntryMeasureBodyRetain( TableRetainIds & ids, const FleetTiersEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool FleetTiersEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const FleetTiersEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool FleetTiersEntryLoadBodyRetain( TableReader & r, FleetTiersEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool FleetTiersEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, FleetTiersEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t FleetMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Fleet & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FleetSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Fleet & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FleetSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Fleet & value, TableRetain * retain, const TableRetainPath & path ); inline bool FleetLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Fleet & value, TableRetain * retain, const TableRetainPath & path ); +inline bool FleetLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Fleet & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t ShipConfigMeasureBody( TableIds & ids, const ShipConfig & value ) { @@ -12661,80 +13101,197 @@ MAPDEMO_TABLE_INLINE bool ShipConfigLoadBodyRetain( TableReader & r, ShipConfig } } -inline int64_t ItemMeasureBodyRetain( TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.count != 0 ) { bytes += TableLebBytes( ids.ref( 0xb1e5e28e4479a274ull ) ) + 1 + 4; } // count - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -MAPDEMO_TABLE_INLINE bool ItemSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( value.count != 0 ) - { - w.putleb( ids.ref( 0xb1e5e28e4479a274ull ) ); w.put8( 4 ); // count - w.put32( uint32_t( value.count ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -MAPDEMO_TABLE_INLINE bool ItemLoadBodyRetain( TableReader & r, Item & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ShipConfigLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, ShipConfig & value, TableRetain * retain, const TableRetainPath & path ) { - ItemReset( value ); // prefill declared defaults in place, then overlay + ShipConfigReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count - const uint64_t field_id = r.ids->at( field_ref ); - if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } - uint8_t kind = r.get8(); - if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) - { - // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, - // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node - // table's is the ROOT body's alone, on the numbering's own - // rule — a second numbering cannot exist — and the BUILD - // VERSION's rides in the announcement and nowhere else. That - // body stops and the parent reads on past its L. - r.report->malformed = true; - return false; - } - switch ( field_id ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - case 0xb1e5e28e4479a274ull: // count + case 0xc4bcadba8e631b86ull: // name { - if ( kind != 4 ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) { - if ( TableKindWidens( kind, 4 ) ) - { - // WIDENED (§4): a kind that grew since the writer decodes - // exactly at its own width, the value lands, one widened counts - int64_t widened_v = 0; - if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } - int32_t decoded_v = (int32_t) widened_v; - value.count = decoded_v; - r.report->widened++; - break; - } - // AT A POSITION THE READER DOES NAME, a field under - // kind 31 or kind 32 takes this same rule and no other (§3) - r.report->kind_mismatch++; - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } break; } - if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } - int32_t decoded_v = int32_t( r.get32( ) ); - value.count = decoded_v; + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 64 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.name, text, (size_t) kept ); + value.name[kept] = 0; + r.offset += (int64_t) n * 8; + value.name_length = kept; + } + break; + } + case 0x7f69d4b5288ba9cfull: // health + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.health = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.health = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t ItemMeasureBodyRetain( TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.count != 0 ) { bytes += TableLebBytes( ids.ref( 0xb1e5e28e4479a274ull ) ) + 1 + 4; } // count + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +MAPDEMO_TABLE_INLINE bool ItemSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Item & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.count != 0 ) + { + w.putleb( ids.ref( 0xb1e5e28e4479a274ull ) ); w.put8( 4 ); // count + w.put32( uint32_t( value.count ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +MAPDEMO_TABLE_INLINE bool ItemLoadBodyRetain( TableReader & r, Item & value, TableRetain * retain, const TableRetainPath & path ) +{ + ItemReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0xb1e5e28e4479a274ull: // count + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.count = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.count = decoded_v; break; } default: @@ -12747,6 +13304,98 @@ MAPDEMO_TABLE_INLINE bool ItemLoadBodyRetain( TableReader & r, Item & value, Tab } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ItemLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Item & value, TableRetain * retain, const TableRetainPath & path ) +{ + ItemReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xb1e5e28e4479a274ull: // count + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.count = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.count = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + inline int64_t FleetShipsEntryMeasureBodyRetain( TableRetainIds & ids, const FleetShipsEntry & value, TableRetain * retain, const TableRetainPath & path ) { int64_t bytes = 1; // the ZERO REFERENCE that ends the body @@ -12878,26 +13527,108 @@ MAPDEMO_TABLE_INLINE bool FleetShipsEntryLoadBodyRetain( TableReader & r, FleetS } } -template -inline int64_t FleetByIdEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const FleetByIdEntry & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool FleetShipsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, FleetShipsEntry & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + FleetShipsEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - const ShipConfig * pointee_value = ShipConfigAt( ctx, value.value ); // *ShipConfig - // A POINTER RIDES AS A NODE INDEX (docs/SPEC-TABLES.md §3.1): the - // header and the index and nothing below it, because the pointee's - // body is in the node table and not here. NULL IS ELIDED — absence - // and null are one value — and a non-null pointer ALWAYS rides, even - // when its node's body is entirely default. - if ( pointee_value != NULL ) - { - uint64_t index_value = 0; - if ( !TableNumberingIndex( numbering, (const void *) pointee_value, index_value ) ) { return -1; } - bytes += TableLebBytes( ids.ref( 0x7ce4fd9430e80ceaull ) ) + 1 + TableLebBytes( index_value ); - } - } - bytes += TableRetainTailMeasure( retain, ids, path ); + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 32 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); + value.key[kept] = 0; + r.offset += (int64_t) n * 8; + value.key_length = kept; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ShipConfigLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t FleetByIdEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const FleetByIdEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + const ShipConfig * pointee_value = ShipConfigAt( ctx, value.value ); // *ShipConfig + // A POINTER RIDES AS A NODE INDEX (docs/SPEC-TABLES.md §3.1): the + // header and the index and nothing below it, because the pointee's + // body is in the node table and not here. NULL IS ELIDED — absence + // and null are one value — and a non-null pointer ALWAYS rides, even + // when its node's body is entirely default. + if ( pointee_value != NULL ) + { + uint64_t index_value = 0; + if ( !TableNumberingIndex( numbering, (const void *) pointee_value, index_value ) ) { return -1; } + bytes += TableLebBytes( ids.ref( 0x7ce4fd9430e80ceaull ) ) + 1 + TableLebBytes( index_value ); + } + } + bytes += TableRetainTailMeasure( retain, ids, path ); return bytes; } @@ -13022,6 +13753,105 @@ inline bool FleetByIdEntryLoadBodyRetain( TableReader & r, const TableNodeMap & } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool FleetByIdEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, FleetByIdEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + FleetByIdEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.value, node_index, 0x758252d2d1b14f0dull, report ); // *ShipConfig + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + inline int64_t FleetLoadoutsEntryValueEntryMeasureBodyRetain( TableRetainIds & ids, const FleetLoadoutsEntryValueEntry & value, TableRetain * retain, const TableRetainPath & path ) { int64_t bytes = 1; // the ZERO REFERENCE that ends the body @@ -13142,6 +13972,86 @@ MAPDEMO_TABLE_INLINE bool FleetLoadoutsEntryValueEntryLoadBodyRetain( TableReade } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool FleetLoadoutsEntryValueEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, FleetLoadoutsEntryValueEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + FleetLoadoutsEntryValueEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 6 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 255ull ) { decoded_wide = (int64_t) 255ull; report->clamped++; } + uint8_t decoded_v = (uint8_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t FleetLoadoutsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const FleetLoadoutsEntry & value, TableRetain * retain, const TableRetainPath & path ) { @@ -13378,16 +14288,149 @@ inline bool FleetLoadoutsEntryLoadBodyRetain( TableReader & r, const TableNodeMa } } -inline int64_t FleetTiersEntryMeasureBodyRetain( TableRetainIds & ids, const FleetTiersEntry & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool FleetLoadoutsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, FleetLoadoutsEntry & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 2; } // key + (void) nodes; (void) index_bits; + FleetLoadoutsEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - const int32_t mark_value = ids.count; - const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); - const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); - if ( body_value < 0 ) { return -1; } - if ( body_value > 1 ) { bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); } // value + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 16 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); + value.key[kept] = 0; + r.offset += (int64_t) n * 8; + value.key_length = kept; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 1 ); + TableMapFill fill = TableMapFillBegin( nodes, value.value, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint8_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const FleetLoadoutsEntryValueEntryMessageKeyRead read = FleetLoadoutsEntryValueEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + FleetLoadoutsEntryValueEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !FleetLoadoutsEntryValueEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 1, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t FleetTiersEntryMeasureBodyRetain( TableRetainIds & ids, const FleetTiersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 2; } // key + { + const int32_t mark_value = ids.count; + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); + if ( body_value < 0 ) { return -1; } + if ( body_value > 1 ) { bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); } // value else { ids.truncate( mark_value ); } // an all-default nested table elides, and costs no entry } bytes += TableRetainTailMeasure( retain, ids, path ); @@ -13509,6 +14552,113 @@ MAPDEMO_TABLE_INLINE bool FleetTiersEntryLoadBodyRetain( TableReader & r, FleetT } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool FleetTiersEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, FleetTiersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + FleetTiersEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 3 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 3 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -32768ll ) { decoded_wide = -32768ll; report->clamped++; } + if ( decoded_wide > 32767ll ) { decoded_wide = 32767ll; report->clamped++; } + int16_t decoded_v = (int16_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -32768ll ) { decoded_wide = -32768ll; report->clamped++; } + if ( decoded_wide > 32767ll ) { decoded_wide = 32767ll; report->clamped++; } + int16_t decoded_v = (int16_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t FleetMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Fleet & value, TableRetain * retain, const TableRetainPath & path ) { @@ -14024,131 +15174,452 @@ inline bool FleetLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Fl FleetLoadoutsEntry * slot = NULL; if ( order == 0 ) { - // EQUAL: a DUPLICATE. The slot that entry took is reset to the - // entry's defaults by the decode below, so LAST WINS WHOLE and an - // elided field of the repeat reads as its default. The map's - // count excludes it. + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + FleetLoadoutsEntryLoadBodyRetain( elem, nodes, *slot, retain, TableRetainStepInto( path, 3, (uint32_t) ( fill.map->count - 1 ) ) ); + } + last_key = read.key; last_length = read.length; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0x6dd8dc6c5fdae3ceull: // tiers + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 4 ); + TableMapFill fill = TableMapFillBegin( nodes, value.tiers, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + int16_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + FleetTiersEntryKeyRead read = FleetTiersEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (int64_t) last_key, (int64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + FleetTiersEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + FleetTiersEntryLoadBodyRetain( elem, *slot, retain, TableRetainStepInto( path, 4, (uint32_t) ( fill.map->count - 1 ) ) ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool FleetLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Fleet & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + FleetReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x294a5c4913e1ad44ull: // ships + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.ships, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + const char * last_key = NULL; int32_t last_length = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const FleetShipsEntryMessageKeyRead read = FleetShipsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( last_key, last_length, read.key, read.length ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + FleetShipsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !FleetShipsEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; last_length = read.length; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0x7b024c46e98d3404ull: // by_id + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 1 ); + TableMapFill fill = TableMapFillBegin( nodes, value.by_id, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const FleetByIdEntryMessageKeyRead read = FleetByIdEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + FleetByIdEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !FleetByIdEntryLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *slot, retain, TableRetainStepInto( path, 1, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0x63dfa0c4a4b3815dull: // flagship + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.flagship, node_index, 0x758252d2d1b14f0dull, report ); // *ShipConfig + } + break; + } + case 0x294fa1b3f0f5f070ull: // loadouts + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 3 ); + TableMapFill fill = TableMapFillBegin( nodes, value.loadouts, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + const char * last_key = NULL; int32_t last_length = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const FleetLoadoutsEntryMessageKeyRead read = FleetLoadoutsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( last_key, last_length, read.key, read.length ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + FleetLoadoutsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. slot = TableMapFillLast( fill ); - r.report->duplicate++; + report->duplicate++; } else { slot = TableMapFillNext( fill ); // ASCENDING: the next slot } - if ( slot == NULL ) { r.report->malformed = true; break; } - { - TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); - FleetLoadoutsEntryLoadBodyRetain( elem, nodes, *slot, retain, TableRetainStepInto( path, 3, (uint32_t) ( fill.map->count - 1 ) ) ); - } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !FleetLoadoutsEntryLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *slot, retain, TableRetainStepInto( path, 3, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends last_key = read.key; last_length = read.length; // the WIRE keys of the entries that LAND landed = true; } TableMapFillEnd( fill ); } - r.offset = body_end; // the remaining entries skip by the map's L break; } case 0x6dd8dc6c5fdae3ceull: // tiers { - if ( kind != 14 ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) { - // AT A POSITION THE READER DOES NAME, a field under - // kind 31 or kind 32 takes this same rule and no other (§3) - r.report->kind_mismatch++; - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } break; } - uint64_t body_len = 0; - if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } - int64_t body_end = r.offset + (int64_t) body_len; - if ( body_len >= 2 ) { - uint8_t elem_kind = r.get8(); uint64_t count = 0; - if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } - // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array - // kind mismatch of §4, and nothing about a map is special-cased - if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the // records under this field go with the value it is about to lose. TableRetainDiscardField( retain, path, 4 ); TableMapFill fill = TableMapFillBegin( nodes, value.tiers, (uint32_t) count ); - if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } - TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree int16_t last_key = 0; bool landed = false; bool map_widened = false; for ( uint64_t i = 0; i < count; i++ ) { - uint64_t elem_len = 0; - if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } - const uint8_t * elem_body = sub.buffer + sub.offset; - sub.offset += (int64_t) elem_len; - FleetTiersEntryKeyRead read = FleetTiersEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + const FleetTiersEntryMessageKeyRead read = FleetTiersEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) - if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } - // THE KEY KIND IS CHECKED FIRST: a key read under another kind - // desynchronizes the rest of the scan, and the honest answer to a - // body whose key is not this reader's kind is the KIND, not the - // framing damage that follows from it. + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } if ( read.kind_bad ) { - // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets - // to EMPTY, ONE kind_mismatch is counted for it, and the rest - // is skipped. Events counted inside earlier entries stand. - r.report->kind_mismatch++; + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } break; } - if ( read.malformed ) { r.report->malformed = true; break; } - if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry const int order = landed ? TableKeyOrder( (int64_t) last_key, (int64_t) read.key ) : -1; - if ( order > 0 ) - { - // DESCENDING: not a body any conforming writer produced. The map - // keeps the ascending prefix it has, the rest skips by the map's - // L, and the PARENT reads on past the field's length (§4). - r.report->malformed = true; - break; - } + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced FleetTiersEntry * slot = NULL; if ( order == 0 ) { - // EQUAL: a DUPLICATE. The slot that entry took is reset to the - // entry's defaults by the decode below, so LAST WINS WHOLE and an - // elided field of the repeat reads as its default. The map's - // count excludes it. + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. slot = TableMapFillLast( fill ); - r.report->duplicate++; + report->duplicate++; } else { slot = TableMapFillNext( fill ); // ASCENDING: the next slot } - if ( slot == NULL ) { r.report->malformed = true; break; } - { - TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); - FleetTiersEntryLoadBodyRetain( elem, *slot, retain, TableRetainStepInto( path, 4, (uint32_t) ( fill.map->count - 1 ) ) ); - } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !FleetTiersEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 4, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends last_key = read.key; // the WIRE keys of the entries that LAND landed = true; } TableMapFillEnd( fill ); } - r.offset = body_end; // the remaining entries skip by the map's L - break; - } - case 0xffffffffffffffffull: - { - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } break; } default: + report->unknown++; { - r.report->unknown++; - if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } - break; + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); } + break; } } } @@ -14326,6 +15797,182 @@ inline const Fleet * FleetLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// FleetNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool FleetNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + FleetNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !FleetNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x758252d2d1b14f0dull: ok = ShipConfigLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(ShipConfig *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // ShipConfig + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// FleetLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool FleetLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Fleet * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !FleetMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = FleetNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + FleetNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !FleetMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Fleet ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x0a53e00afba279afull; + Fleet * root = new ( region + used ) Fleet; // lifetime only: LoadMessageBody's first act is FleetReset + FleetReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Fleet ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !FleetNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return FleetLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// FleetLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool FleetLoadRetainMessages( const Fleet ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !FleetLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // FleetMeasureRetain and FleetSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/PairsTable.h b/testdata/golden/tables/maps/PairsTable.h index b9b102c4d..5afff9b5d 100644 --- a/testdata/golden/tables/maps/PairsTable.h +++ b/testdata/golden/tables/maps/PairsTable.h @@ -4400,6 +4400,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6237,10 +6669,12 @@ template inline int64_t PairsSlotsEntryMeasureBodyRetain( const C template inline bool PairsSlotsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool PairsSlotsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); inline bool PairsSlotsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PairsSlotsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t PairsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); template inline bool PairsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); template inline bool PairsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); inline bool PairsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Pairs & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PairsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Pairs & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t PairsSlotsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const PairsSlotsEntry & value ) @@ -8583,92 +9017,210 @@ inline bool PairsSlotsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & } } -template -inline int64_t PairsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // slots: a kind 14 array of kind 13 elements, ASCENDING (§2.8) - TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); - if ( !order_slots.ok ) { return -1; } // the sort could not run - if ( order_slots.count > 0 ) - { - const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); - int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); // the element kind byte and the count - for ( int32_t i = 0; i < order_slots.count; i++ ) - { - const int64_t elem_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return -1; } - body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key - } - bytes += TableLebBytes( ref_slots ) + 1 + TableLebBytes( (uint64_t) ( body_slots ) ) + ( body_slots ); - } - TableMapRelease( order_slots ); - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool PairsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) -{ - { - TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots - if ( !order_slots.ok ) { return false; } - if ( order_slots.count > 0 ) // an EMPTY map elides, the by-value rule (§3) - { - const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); - int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); - for ( int32_t i = 0; i < order_slots.count; i++ ) - { - const int64_t elem_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return false; } - body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); - } - w.putleb( ref_slots ); w.put8( 14 ); w.putleb( (uint64_t) body_slots ); - w.put8( 13 ); w.putleb( (uint64_t) order_slots.count ); - for ( int32_t i = 0; i < order_slots.count; i++ ) - { - const int64_t elem_len_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_len_slots < 0 ) { TableMapRelease( order_slots ); return false; } - w.putleb( (uint64_t) elem_len_slots ); - if ( !PairsSlotsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_slots ); return false; } - } - } - TableMapRelease( order_slots ); - } - if ( value.after != 0 ) - { - w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after - w.put32( uint32_t( value.after ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - return !w.overflow; -} - -template -inline bool PairsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( !PairsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -inline bool PairsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Pairs & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool PairsSlotsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) { - PairsReset( value ); // prefill declared defaults in place, then overlay + (void) nodes; (void) index_bits; + PairsSlotsEntryReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.value[i] : scratch ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t PairsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // slots: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); + if ( !order_slots.ok ) { return -1; } // the sort could not run + if ( order_slots.count > 0 ) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return -1; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_slots ) + 1 + TableLebBytes( (uint64_t) ( body_slots ) ) + ( body_slots ); + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool PairsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return false; } + if ( order_slots.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return false; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); + } + w.putleb( ref_slots ); w.put8( 14 ); w.putleb( (uint64_t) body_slots ); + w.put8( 13 ); w.putleb( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_len_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_slots < 0 ) { TableMapRelease( order_slots ); return false; } + w.putleb( (uint64_t) elem_len_slots ); + if ( !PairsSlotsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_slots ); return false; } + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool PairsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !PairsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + PairsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count const uint64_t field_id = r.ids->at( field_ref ); if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } uint8_t kind = r.get8(); @@ -8816,6 +9368,164 @@ inline bool PairsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Pa } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool PairsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + PairsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xe68c2e6bb1ee5646ull: // slots + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.slots, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const PairsSlotsEntryMessageKeyRead read = PairsSlotsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + PairsSlotsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !PairsSlotsEntryLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // PairsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8989,6 +9699,182 @@ inline const Pairs * PairsLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// PairsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool PairsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + PairsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !PairsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// PairsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool PairsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Pairs * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !PairsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = PairsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + PairsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !PairsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x1404200dab337086ull; + Pairs * root = new ( region + used ) Pairs; // lifetime only: LoadMessageBody's first act is PairsReset + PairsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !PairsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return PairsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// PairsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool PairsLoadRetainMessages( const Pairs ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !PairsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // PairsMeasureRetain and PairsSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/RowsTable.h b/testdata/golden/tables/maps/RowsTable.h index 6c76612df..a452b9feb 100644 --- a/testdata/golden/tables/maps/RowsTable.h +++ b/testdata/golden/tables/maps/RowsTable.h @@ -4400,6 +4400,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6764,27 +7196,34 @@ inline EdgeRowIdsEntryKeyRead EdgeRowIdsEntryReadKey( const uint8_t * body, int6 inline int64_t RowEntriesEntryMeasureBodyRetain( TableRetainIds & ids, const RowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool RowEntriesEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const RowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool RowEntriesEntryLoadBodyRetain( TableReader & r, RowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool RowEntriesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, RowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t RowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Row & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RowSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Row & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RowSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Row & value, TableRetain * retain, const TableRetainPath & path ); inline bool RowLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Row & value, TableRetain * retain, const TableRetainPath & path ); +inline bool RowLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Row & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t WideRowEntriesEntryMeasureBodyRetain( TableRetainIds & ids, const WideRowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool WideRowEntriesEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const WideRowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool WideRowEntriesEntryLoadBodyRetain( TableReader & r, WideRowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool WideRowEntriesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, WideRowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t WideRowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const WideRow & value, TableRetain * retain, const TableRetainPath & path ); template inline bool WideRowSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const WideRow & value, TableRetain * retain, const TableRetainPath & path ); template inline bool WideRowSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const WideRow & value, TableRetain * retain, const TableRetainPath & path ); inline bool WideRowLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, WideRow & value, TableRetain * retain, const TableRetainPath & path ); +inline bool WideRowLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, WideRow & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t EdgeRowNamesEntryMeasureBodyRetain( TableRetainIds & ids, const EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool EdgeRowNamesEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool EdgeRowNamesEntryLoadBodyRetain( TableReader & r, EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool EdgeRowNamesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t EdgeRowIdsEntryMeasureBodyRetain( TableRetainIds & ids, const EdgeRowIdsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool EdgeRowIdsEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const EdgeRowIdsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool EdgeRowIdsEntryLoadBodyRetain( TableReader & r, EdgeRowIdsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool EdgeRowIdsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, EdgeRowIdsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t EdgeRowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const EdgeRow & value, TableRetain * retain, const TableRetainPath & path ); template inline bool EdgeRowSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const EdgeRow & value, TableRetain * retain, const TableRetainPath & path ); template inline bool EdgeRowSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const EdgeRow & value, TableRetain * retain, const TableRetainPath & path ); inline bool EdgeRowLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, EdgeRow & value, TableRetain * retain, const TableRetainPath & path ); +inline bool EdgeRowLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, EdgeRow & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t RowEntriesEntryMeasureBody( TableIds & ids, const RowEntriesEntry & value ) { @@ -13023,6 +13462,88 @@ MAPDEMO_TABLE_INLINE bool RowEntriesEntryLoadBodyRetain( TableReader & r, RowEnt } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool RowEntriesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, RowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + RowEntriesEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); + value.key[kept] = 0; + r.offset += (int64_t) n * 8; + value.key_length = kept; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t RowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Row & value, TableRetain * retain, const TableRetainPath & path ) { @@ -13259,15 +13780,173 @@ inline bool RowLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Row } } -inline int64_t WideRowEntriesEntryMeasureBodyRetain( TableRetainIds & ids, const WideRowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool RowLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Row & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + (void) nodes; (void) index_bits; + RowReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - const int32_t mark_value = ids.count; - const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); - const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); - if ( body_value < 0 ) { return -1; } + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xc5b2a72c0845a253ull: // entries + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.entries, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + const char * last_key = NULL; int32_t last_length = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const RowEntriesEntryMessageKeyRead read = RowEntriesEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( last_key, last_length, read.key, read.length ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + RowEntriesEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !RowEntriesEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; last_length = read.length; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t WideRowEntriesEntryMeasureBodyRetain( TableRetainIds & ids, const WideRowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + const int32_t mark_value = ids.count; + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); + if ( body_value < 0 ) { return -1; } if ( body_value > 1 ) { bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); } // value else { ids.truncate( mark_value ); } // an all-default nested table elides, and costs no entry } @@ -13390,6 +14069,101 @@ MAPDEMO_TABLE_INLINE bool WideRowEntriesEntryLoadBodyRetain( TableReader & r, Wi } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool WideRowEntriesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, WideRowEntriesEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + WideRowEntriesEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t WideRowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const WideRow & value, TableRetain * retain, const TableRetainPath & path ) { @@ -13626,26 +14400,184 @@ inline bool WideRowLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, } } -inline int64_t EdgeRowNamesEntryMeasureBodyRetain( TableRetainIds & ids, const EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool WideRowLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, WideRow & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.key_length < 0 || value.key_length > 300 ) { return -1; } // storage invariant - if ( value.key_length > 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + TableLebBytes( (uint64_t) ( value.key_length ) ) + ( value.key_length ); } // key + (void) nodes; (void) index_bits; + WideRowReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - const int32_t mark_value = ids.count; - const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); - const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); - if ( body_value < 0 ) { return -1; } - if ( body_value > 1 ) { bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); } // value - else { ids.truncate( mark_value ); } // an all-default nested table elides, and costs no entry - } - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -MAPDEMO_TABLE_INLINE bool EdgeRowNamesEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( value.key_length < 0 || value.key_length > 300 ) { return false; } // storage invariant + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xc5b2a72c0845a253ull: // entries + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.entries, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const WideRowEntriesEntryMessageKeyRead read = WideRowEntriesEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + WideRowEntriesEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !WideRowEntriesEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t EdgeRowNamesEntryMeasureBodyRetain( TableRetainIds & ids, const EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key_length < 0 || value.key_length > 300 ) { return -1; } // storage invariant + if ( value.key_length > 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + TableLebBytes( (uint64_t) ( value.key_length ) ) + ( value.key_length ); } // key + { + const int32_t mark_value = ids.count; + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + const int64_t body_value = ItemMeasureBodyRetain( ids, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); + if ( body_value < 0 ) { return -1; } + if ( body_value > 1 ) { bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); } // value + else { ids.truncate( mark_value ); } // an all-default nested table elides, and costs no entry + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +MAPDEMO_TABLE_INLINE bool EdgeRowNamesEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.key_length < 0 || value.key_length > 300 ) { return false; } // storage invariant if ( value.key_length > 0 ) { w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 12 ); // key @@ -13757,6 +14689,88 @@ MAPDEMO_TABLE_INLINE bool EdgeRowNamesEntryLoadBodyRetain( TableReader & r, Edge } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool EdgeRowNamesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, EdgeRowNamesEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + EdgeRowNamesEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 300 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); + value.key[kept] = 0; + r.offset += (int64_t) n * 8; + value.key_length = kept; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + inline int64_t EdgeRowIdsEntryMeasureBodyRetain( TableRetainIds & ids, const EdgeRowIdsEntry & value, TableRetain * retain, const TableRetainPath & path ) { int64_t bytes = 1; // the ZERO REFERENCE that ends the body @@ -13888,19 +14902,112 @@ MAPDEMO_TABLE_INLINE bool EdgeRowIdsEntryLoadBodyRetain( TableReader & r, EdgeRo } } -template -inline int64_t EdgeRowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const EdgeRow & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool EdgeRowIdsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, EdgeRowIdsEntry & value, TableRetain * retain, const TableRetainPath & path ) { - (void) ctx; (void) numbering; - int64_t bytes = 1; // the ZERO REFERENCE that ends the body + EdgeRowIdsEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - // names: a kind 14 array of kind 13 elements, ASCENDING (§2.8) - TableMapCursor order_names = TableMapOrder( ctx, value.names ); - if ( !order_names.ok ) { return -1; } // the sort could not run - if ( order_names.count > 0 ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - const uint64_t ref_names = ids.ref( 0xafb728fff268814full ); - int64_t body_names = 1 + TableLebBytes( (uint64_t) order_names.count ); // the element kind byte and the count + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 9 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 9 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + uint64_t decoded_v = (uint64_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + uint64_t decoded_v = (uint64_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.value, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t EdgeRowMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const EdgeRow & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // names: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_names = TableMapOrder( ctx, value.names ); + if ( !order_names.ok ) { return -1; } // the sort could not run + if ( order_names.count > 0 ) + { + const uint64_t ref_names = ids.ref( 0xafb728fff268814full ); + int64_t body_names = 1 + TableLebBytes( (uint64_t) order_names.count ); // the element kind byte and the count for ( int32_t i = 0; i < order_names.count; i++ ) { const int64_t elem_names = EdgeRowNamesEntryMeasureBodyRetain( ids, *order_names[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); @@ -14257,177 +15364,576 @@ inline bool EdgeRowLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, } } -// RowNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void RowNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) -{ - // the node's own EXTENT, where its lists' and maps' arrays are carved - // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). - // The tool's path carries a worker instead: there the arrays are the - // arena's. - TableExtentCarve carve; - carve.worker = nodes.worker; - if ( carve.worker == NULL ) - { - TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse - const int64_t storage = RowNodeStorage( type_id, r.size, reason ); - const int64_t record = storage > 0 ? RowNodeRecordBytes( type_id ) : 0; - carve.at = at + record; - carve.left = storage > record ? storage - record : 0; - } - nodes.carve = &carve; - (void) nodes; // every node this root can name is a FIXED table - (void) retain; (void) node; - switch ( type_id ) - { - default: break; - } - nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done -} - -// RowLoadRetain: decode the tolerant wire into the caller's exact-sized region and -// return the root. LOAD IS A SCAN, and that is the whole of its bound: it -// follows no reference, so there is no depth cap, no visited set and no -// ordering rule on the indices. Partial results are kept, as everywhere on -// this wire — the report says what happened. NULL means the CALLER's buffer -// was wrong. -// UNDER RETENTION it also fills the caller's two stores with the fields -// this build cannot name, and the report carries what it could not keep -// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the -// reader's data is exactly what it would have been with retention off. -inline const Row * RowLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool EdgeRowLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, EdgeRow & value, TableRetain * retain, const TableRetainPath & path ) { - TableReport ignored; - TableReport * out = report != NULL ? report : &ignored; - // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: - // a file that is both a newer form and damaged is a REFUSAL and never - // damage (docs/SPEC-TABLES.md §3). - TableIdTable ids_table; - int64_t body_bytes = 0; - const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); - if ( verdict != TableOpenOk ) - { - if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } - return NULL; - } - if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) - { - out->malformed = true; // a byte no field claims, before the table (§3) - return NULL; - } - const uint8_t * const wire = wire_file + 1; - const int64_t wire_bytes = body_bytes; - if ( region == NULL || region_bytes < (int64_t) sizeof( Row ) ) { out->malformed = true; return NULL; } - if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } - memset( region, 0, (size_t) region_bytes ); - uint64_t type_id = 0; - const uint8_t * body = NULL; - int64_t length = 0; - - // the record count and the data bytes, from the FRAMING alone - TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed - int64_t root_extent = 0; - if ( !RowWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } - int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Row ) ) + root_extent ); - int64_t records = 0; - { - TableReport counting; - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); - while ( TableNodeScanNext( scan, type_id, body, length ) ) - { - records++; - int64_t storage = RowNodeStorage( type_id, length, reason ); - if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } - if ( storage > 0 ) { data += storage; } - } - } - int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); - if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } - - TableNodeMap nodes; - nodes.base = region; - nodes.entries = (const TableNodeDirEntry *) ( region + data ); - nodes.count = records + 1; - TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); - directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) - directory[0].type_id = 0xa013e119fec906fbull; - Row * root = new ( region ) Row; // lifetime only: LoadBody's first act is RowReset - RowReset( *root ); - - // LoadRetain RESETS BOTH STORES and writes into neither id list: a - // retained record carries its field's identity in the record itself, - // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer - // belongs to this region from here on. - TableRetainReset( retain, nodes, region ); - - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Row ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) - { - int64_t storage = RowNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - RowNodePlace( type_id, region + used, length ); - used += storage; - } - k++; - } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) + (void) nodes; (void) index_bits; + EdgeRowReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) + case 0xafb728fff268814full: // names { - TableReader sub( body, length, out, &ids_table ); - RowNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; - } - } - - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.names, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + const char * last_key = NULL; int32_t last_length = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const EdgeRowNamesEntryMessageKeyRead read = EdgeRowNamesEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( last_key, last_length, read.key, read.length ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + EdgeRowNamesEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !EdgeRowNamesEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; last_length = read.length; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0x2b7dea192bb7be29ull: // ids + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 1 ); + TableMapFill fill = TableMapFillBegin( nodes, value.ids, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint64_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const EdgeRowIdsEntryMessageKeyRead read = EdgeRowIdsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + EdgeRowIdsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !EdgeRowIdsEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 1, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// RowNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void RowNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = RowNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? RowNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + (void) retain; (void) node; + switch ( type_id ) + { + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// RowLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Row * RowLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Row ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !RowWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Row ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = RowNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xa013e119fec906fbull; + Row * root = new ( region ) Row; // lifetime only: LoadBody's first act is RowReset + RowReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Row ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = RowNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + RowNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + RowNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Row ) ); root_carve.left = root_extent; nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - RowLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + RowLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// RowNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool RowNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + RowNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !RowNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// RowLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool RowLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Row * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !RowMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = RowNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + RowNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !RowMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Row ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xa013e119fec906fbull; + Row * root = new ( region + used ) Row; // lifetime only: LoadMessageBody's first act is RowReset + RowReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Row ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !RowNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return RowLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// RowLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool RowLoadRetainMessages( const Row ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !RowLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // RowMeasureRetain and RowSaveRetain: the pair, with the retained tail in @@ -14651,71 +16157,247 @@ inline const WideRow * WideRowLoadRetain( uint8_t * region, int64_t region_bytes // belongs to this region from here on. TableRetainReset( retain, nodes, region ); - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( WideRow ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = WideRowNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + WideRowNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + WideRowNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( WideRow ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + WideRowLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// WideRowNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool WideRowNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + WideRowNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !WideRowNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// WideRowLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool WideRowLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const WideRow * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !WideRowMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = WideRowNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + WideRowNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !WideRowMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( WideRow ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xb413964e3571a316ull; + WideRow * root = new ( region + used ) WideRow; // lifetime only: LoadMessageBody's first act is WideRowReset + WideRowReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( WideRow ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( WideRow ) ) + root_extent ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) { - int64_t storage = WideRowNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else - { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - WideRowNodePlace( type_id, region + used, length ); - used += storage; - } - k++; + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) + if ( directory[k + 1].offset == kTableNodeAbsent ) { - if ( directory[k + 1].offset != kTableNodeAbsent ) - { - TableReader sub( body, length, out, &ids_table ); - WideRowNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); - } - k++; + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; } + if ( !WideRowNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TableExtentCarve root_carve; - root_carve.at = region + TableAlignUp64( (int64_t) sizeof( WideRow ) ); - root_carve.left = root_extent; + // and the ROOT's own body last nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's - WideRowLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + return WideRowLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// WideRowLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool WideRowLoadRetainMessages( const WideRow ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !WideRowLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // WideRowMeasureRetain and WideRowSaveRetain: the pair, with the retained tail in @@ -15006,6 +16688,182 @@ inline const EdgeRow * EdgeRowLoadRetain( uint8_t * region, int64_t region_bytes return root; } +// EdgeRowNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool EdgeRowNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + EdgeRowNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !EdgeRowNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// EdgeRowLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool EdgeRowLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const EdgeRow * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !EdgeRowMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = EdgeRowNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + EdgeRowNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !EdgeRowMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( EdgeRow ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xe8130af045a036f8ull; + EdgeRow * root = new ( region + used ) EdgeRow; // lifetime only: LoadMessageBody's first act is EdgeRowReset + EdgeRowReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( EdgeRow ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !EdgeRowNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return EdgeRowLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// EdgeRowLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool EdgeRowLoadRetainMessages( const EdgeRow ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !EdgeRowLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // EdgeRowMeasureRetain and EdgeRowSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/RunsTable.h b/testdata/golden/tables/maps/RunsTable.h index 1a17d617e..480b5e5af 100644 --- a/testdata/golden/tables/maps/RunsTable.h +++ b/testdata/golden/tables/maps/RunsTable.h @@ -4400,6 +4400,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6233,10 +6665,12 @@ inline RunsSpansEntryKeyRead RunsSpansEntryReadKey( const uint8_t * body, int64_ inline int64_t RunsSpansEntryMeasureBodyRetain( TableRetainIds & ids, const RunsSpansEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool RunsSpansEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const RunsSpansEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool RunsSpansEntryLoadBodyRetain( TableReader & r, RunsSpansEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool RunsSpansEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, RunsSpansEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t RunsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RunsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ); template inline bool RunsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ); inline bool RunsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Runs & value, TableRetain * retain, const TableRetainPath & path ); +inline bool RunsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Runs & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t RunsSpansEntryMeasureBody( TableIds & ids, const RunsSpansEntry & value ) { @@ -8325,93 +8759,208 @@ MAPDEMO_TABLE_INLINE bool RunsSpansEntryLoadBodyRetain( TableReader & r, RunsSpa } } -template -inline int64_t RunsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ) -{ - (void) ctx; (void) numbering; - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // spans: a kind 14 array of kind 13 elements, ASCENDING (§2.8) - TableMapCursor order_spans = TableMapOrder( ctx, value.spans ); - if ( !order_spans.ok ) { return -1; } // the sort could not run - if ( order_spans.count > 0 ) - { - const uint64_t ref_spans = ids.ref( 0x437dfc8ab2566816ull ); - int64_t body_spans = 1 + TableLebBytes( (uint64_t) order_spans.count ); // the element kind byte and the count - for ( int32_t i = 0; i < order_spans.count; i++ ) - { - const int64_t elem_spans = RunsSpansEntryMeasureBodyRetain( ids, *order_spans[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_spans < 0 ) { TableMapRelease( order_spans ); return -1; } - body_spans += TableLebBytes( (uint64_t) ( elem_spans ) ) + ( elem_spans ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key - } - bytes += TableLebBytes( ref_spans ) + 1 + TableLebBytes( (uint64_t) ( body_spans ) ) + ( body_spans ); - } - TableMapRelease( order_spans ); - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool RunsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ) -{ - (void) ctx; (void) numbering; - { - TableMapCursor order_spans = TableMapOrder( ctx, value.spans ); // spans - if ( !order_spans.ok ) { return false; } - if ( order_spans.count > 0 ) // an EMPTY map elides, the by-value rule (§3) - { - const uint64_t ref_spans = ids.ref( 0x437dfc8ab2566816ull ); - int64_t body_spans = 1 + TableLebBytes( (uint64_t) order_spans.count ); - for ( int32_t i = 0; i < order_spans.count; i++ ) - { - const int64_t elem_spans = RunsSpansEntryMeasureBodyRetain( ids, *order_spans[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_spans < 0 ) { TableMapRelease( order_spans ); return false; } - body_spans += TableLebBytes( (uint64_t) ( elem_spans ) ) + ( elem_spans ); - } - w.putleb( ref_spans ); w.put8( 14 ); w.putleb( (uint64_t) body_spans ); - w.put8( 13 ); w.putleb( (uint64_t) order_spans.count ); - for ( int32_t i = 0; i < order_spans.count; i++ ) - { - const int64_t elem_len_spans = RunsSpansEntryMeasureBodyRetain( ids, *order_spans[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_len_spans < 0 ) { TableMapRelease( order_spans ); return false; } - w.putleb( (uint64_t) elem_len_spans ); - if ( !RunsSpansEntrySaveBodyRetain( w, ids, *order_spans[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_spans ); return false; } - } - } - TableMapRelease( order_spans ); - } - if ( value.after != 0 ) - { - w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after - w.put32( uint32_t( value.after ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - return !w.overflow; -} - -template -inline bool RunsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( !RunsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -inline bool RunsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Runs & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool RunsSpansEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, RunsSpansEntry & value, TableRetain * retain, const TableRetainPath & path ) { - (void) nodes; - RunsReset( value ); // prefill declared defaults in place, then overlay + RunsSpansEntryReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 7 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 7 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 65535ull ) { decoded_wide = (int64_t) 65535ull; report->clamped++; } + uint16_t decoded_v = (uint16_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 65535ull ) { decoded_wide = (int64_t) 65535ull; report->clamped++; } + uint16_t decoded_v = (uint16_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 4 ) { kept = 4; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + Item scratch; + ItemReset( scratch ); + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, ( in_bounds ? value.value[i] : scratch ), ( in_bounds ? retain : NULL ), TableRetainStepInto( path, 1, (uint32_t) ( i ) ) ) ) { return false; } + } + value.value_count = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t RunsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // spans: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_spans = TableMapOrder( ctx, value.spans ); + if ( !order_spans.ok ) { return -1; } // the sort could not run + if ( order_spans.count > 0 ) + { + const uint64_t ref_spans = ids.ref( 0x437dfc8ab2566816ull ); + int64_t body_spans = 1 + TableLebBytes( (uint64_t) order_spans.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_spans.count; i++ ) + { + const int64_t elem_spans = RunsSpansEntryMeasureBodyRetain( ids, *order_spans[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_spans < 0 ) { TableMapRelease( order_spans ); return -1; } + body_spans += TableLebBytes( (uint64_t) ( elem_spans ) ) + ( elem_spans ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_spans ) + 1 + TableLebBytes( (uint64_t) ( body_spans ) ) + ( body_spans ); + } + TableMapRelease( order_spans ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool RunsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + { + TableMapCursor order_spans = TableMapOrder( ctx, value.spans ); // spans + if ( !order_spans.ok ) { return false; } + if ( order_spans.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_spans = ids.ref( 0x437dfc8ab2566816ull ); + int64_t body_spans = 1 + TableLebBytes( (uint64_t) order_spans.count ); + for ( int32_t i = 0; i < order_spans.count; i++ ) + { + const int64_t elem_spans = RunsSpansEntryMeasureBodyRetain( ids, *order_spans[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_spans < 0 ) { TableMapRelease( order_spans ); return false; } + body_spans += TableLebBytes( (uint64_t) ( elem_spans ) ) + ( elem_spans ); + } + w.putleb( ref_spans ); w.put8( 14 ); w.putleb( (uint64_t) body_spans ); + w.put8( 13 ); w.putleb( (uint64_t) order_spans.count ); + for ( int32_t i = 0; i < order_spans.count; i++ ) + { + const int64_t elem_len_spans = RunsSpansEntryMeasureBodyRetain( ids, *order_spans[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_spans < 0 ) { TableMapRelease( order_spans ); return false; } + w.putleb( (uint64_t) elem_len_spans ); + if ( !RunsSpansEntrySaveBodyRetain( w, ids, *order_spans[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_spans ); return false; } + } + } + TableMapRelease( order_spans ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool RunsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Runs & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !RunsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool RunsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Runs & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; + RunsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count const uint64_t field_id = r.ids->at( field_ref ); @@ -8561,6 +9110,164 @@ inline bool RunsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Run } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool RunsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Runs & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + RunsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x437dfc8ab2566816ull: // spans + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.spans, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint16_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const RunsSpansEntryMessageKeyRead read = RunsSpansEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + RunsSpansEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !RunsSpansEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // RunsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8734,6 +9441,182 @@ inline const Runs * RunsLoadRetain( uint8_t * region, int64_t region_bytes, cons return root; } +// RunsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool RunsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + RunsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !RunsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// RunsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool RunsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Runs * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !RunsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = RunsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + RunsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !RunsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Runs ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xea7bdd2b70c8c2bbull; + Runs * root = new ( region + used ) Runs; // lifetime only: LoadMessageBody's first act is RunsReset + RunsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Runs ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !RunsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return RunsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// RunsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool RunsLoadRetainMessages( const Runs ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !RunsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // RunsMeasureRetain and RunsSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/SlotsTable.h b/testdata/golden/tables/maps/SlotsTable.h index c205575b6..0420547af 100644 --- a/testdata/golden/tables/maps/SlotsTable.h +++ b/testdata/golden/tables/maps/SlotsTable.h @@ -4400,6 +4400,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6298,10 +6730,12 @@ inline SlotsSeatsEntryKeyRead SlotsSeatsEntryReadKey( const uint8_t * body, int6 inline int64_t SlotsSeatsEntryMeasureBodyRetain( TableRetainIds & ids, const SlotsSeatsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool SlotsSeatsEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const SlotsSeatsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool SlotsSeatsEntryLoadBodyRetain( TableReader & r, SlotsSeatsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SlotsSeatsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, SlotsSeatsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t SlotsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SlotsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SlotsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ); inline bool SlotsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Slots & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SlotsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Slots & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t SlotsSeatsEntryMeasureBody( TableIds & ids, const SlotsSeatsEntry & value ) { @@ -8551,93 +8985,232 @@ MAPDEMO_TABLE_INLINE bool SlotsSeatsEntryLoadBodyRetain( TableReader & r, SlotsS } } -template -inline int64_t SlotsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ) -{ - (void) ctx; (void) numbering; - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // seats: a kind 14 array of kind 13 elements, ASCENDING (§2.8) - TableMapCursor order_seats = TableMapOrder( ctx, value.seats ); - if ( !order_seats.ok ) { return -1; } // the sort could not run - if ( order_seats.count > 0 ) - { - const uint64_t ref_seats = ids.ref( 0x7f6548303072b061ull ); - int64_t body_seats = 1 + TableLebBytes( (uint64_t) order_seats.count ); // the element kind byte and the count - for ( int32_t i = 0; i < order_seats.count; i++ ) - { - const int64_t elem_seats = SlotsSeatsEntryMeasureBodyRetain( ids, *order_seats[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_seats < 0 ) { TableMapRelease( order_seats ); return -1; } - body_seats += TableLebBytes( (uint64_t) ( elem_seats ) ) + ( elem_seats ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key - } - bytes += TableLebBytes( ref_seats ) + 1 + TableLebBytes( (uint64_t) ( body_seats ) ) + ( body_seats ); - } - TableMapRelease( order_seats ); - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool SlotsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ) -{ - (void) ctx; (void) numbering; - { - TableMapCursor order_seats = TableMapOrder( ctx, value.seats ); // seats - if ( !order_seats.ok ) { return false; } - if ( order_seats.count > 0 ) // an EMPTY map elides, the by-value rule (§3) - { - const uint64_t ref_seats = ids.ref( 0x7f6548303072b061ull ); - int64_t body_seats = 1 + TableLebBytes( (uint64_t) order_seats.count ); - for ( int32_t i = 0; i < order_seats.count; i++ ) - { - const int64_t elem_seats = SlotsSeatsEntryMeasureBodyRetain( ids, *order_seats[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_seats < 0 ) { TableMapRelease( order_seats ); return false; } - body_seats += TableLebBytes( (uint64_t) ( elem_seats ) ) + ( elem_seats ); - } - w.putleb( ref_seats ); w.put8( 14 ); w.putleb( (uint64_t) body_seats ); - w.put8( 13 ); w.putleb( (uint64_t) order_seats.count ); - for ( int32_t i = 0; i < order_seats.count; i++ ) - { - const int64_t elem_len_seats = SlotsSeatsEntryMeasureBodyRetain( ids, *order_seats[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_len_seats < 0 ) { TableMapRelease( order_seats ); return false; } - w.putleb( (uint64_t) elem_len_seats ); - if ( !SlotsSeatsEntrySaveBodyRetain( w, ids, *order_seats[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_seats ); return false; } - } - } - TableMapRelease( order_seats ); - } - if ( value.after != 0 ) - { - w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after - w.put32( uint32_t( value.after ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - return !w.overflow; -} - -template -inline bool SlotsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( !SlotsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -inline bool SlotsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Slots & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SlotsSeatsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, SlotsSeatsEntry & value, TableRetain * retain, const TableRetainPath & path ) { - (void) nodes; - SlotsReset( value ); // prefill declared defaults in place, then overlay + SlotsSeatsEntryReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 16 || entry.elem_kind != 4 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { report->malformed = true; return false; } + for ( uint64_t p = 0; p < n; p++ ) + { + uint64_t key_ref = 0; + if ( !r.get( key_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry key_entry; + if ( !TableMessageNameEntry( vocabulary, key_ref, key_entry ) ) { report->malformed = true; return false; } + Slot key = Slot::None; + const bool named = TableEnumValue( key_entry.id, key ); + if ( !named ) { report->unknown++; report->retain_lost++; } + const int32_t slot = named ? (int32_t) key - 1 : -1; + const bool in_bounds = slot >= 0 && slot < 2; + int32_t scratch = 0; + { + const int64_t width_2 = entry.elem_value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( entry.elem_packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) entry.elem_base_lo ); } + else if ( width_2 > 0 && width_2 < 64 ) + { + const uint64_t sign_2 = uint64_t(1) << ( width_2 - 1 ); + if ( ( raw_2 & sign_2 ) != 0 ) { decoded_wide_2 = (int64_t) ( raw_2 | ~( ( uint64_t(1) << width_2 ) - 1 ) ); } + } + if ( decoded_wide_2 < -2147483648ll ) { decoded_wide_2 = -2147483648ll; report->clamped++; } + if ( decoded_wide_2 > 2147483647ll ) { decoded_wide_2 = 2147483647ll; report->clamped++; } + int32_t decoded_v_2 = (int32_t) decoded_wide_2; + scratch = decoded_v_2; + } + if ( in_bounds ) { value.value.slots[slot] = scratch; } + } + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t SlotsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // seats: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_seats = TableMapOrder( ctx, value.seats ); + if ( !order_seats.ok ) { return -1; } // the sort could not run + if ( order_seats.count > 0 ) + { + const uint64_t ref_seats = ids.ref( 0x7f6548303072b061ull ); + int64_t body_seats = 1 + TableLebBytes( (uint64_t) order_seats.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_seats.count; i++ ) + { + const int64_t elem_seats = SlotsSeatsEntryMeasureBodyRetain( ids, *order_seats[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_seats < 0 ) { TableMapRelease( order_seats ); return -1; } + body_seats += TableLebBytes( (uint64_t) ( elem_seats ) ) + ( elem_seats ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_seats ) + 1 + TableLebBytes( (uint64_t) ( body_seats ) ) + ( body_seats ); + } + TableMapRelease( order_seats ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool SlotsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) ctx; (void) numbering; + { + TableMapCursor order_seats = TableMapOrder( ctx, value.seats ); // seats + if ( !order_seats.ok ) { return false; } + if ( order_seats.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_seats = ids.ref( 0x7f6548303072b061ull ); + int64_t body_seats = 1 + TableLebBytes( (uint64_t) order_seats.count ); + for ( int32_t i = 0; i < order_seats.count; i++ ) + { + const int64_t elem_seats = SlotsSeatsEntryMeasureBodyRetain( ids, *order_seats[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_seats < 0 ) { TableMapRelease( order_seats ); return false; } + body_seats += TableLebBytes( (uint64_t) ( elem_seats ) ) + ( elem_seats ); + } + w.putleb( ref_seats ); w.put8( 14 ); w.putleb( (uint64_t) body_seats ); + w.put8( 13 ); w.putleb( (uint64_t) order_seats.count ); + for ( int32_t i = 0; i < order_seats.count; i++ ) + { + const int64_t elem_len_seats = SlotsSeatsEntryMeasureBodyRetain( ids, *order_seats[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_seats < 0 ) { TableMapRelease( order_seats ); return false; } + w.putleb( (uint64_t) elem_len_seats ); + if ( !SlotsSeatsEntrySaveBodyRetain( w, ids, *order_seats[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_seats ); return false; } + } + } + TableMapRelease( order_seats ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool SlotsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Slots & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !SlotsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool SlotsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Slots & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; + SlotsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count const uint64_t field_id = r.ids->at( field_ref ); @@ -8787,6 +9360,164 @@ inline bool SlotsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Sl } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SlotsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Slots & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + SlotsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x7f6548303072b061ull: // seats + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.seats, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + int32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const SlotsSeatsEntryMessageKeyRead read = SlotsSeatsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (int64_t) last_key, (int64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + SlotsSeatsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !SlotsSeatsEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // SlotsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -8960,6 +9691,182 @@ inline const Slots * SlotsLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// SlotsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool SlotsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + SlotsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !SlotsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// SlotsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool SlotsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Slots * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !SlotsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = SlotsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + SlotsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !SlotsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Slots ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xf96b15cd3921d4a6ull; + Slots * root = new ( region + used ) Slots; // lifetime only: LoadMessageBody's first act is SlotsReset + SlotsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Slots ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !SlotsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return SlotsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// SlotsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool SlotsLoadRetainMessages( const Slots ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !SlotsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // SlotsMeasureRetain and SlotsSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/SpansTable.h b/testdata/golden/tables/maps/SpansTable.h index 1014e660d..caa2c073d 100644 --- a/testdata/golden/tables/maps/SpansTable.h +++ b/testdata/golden/tables/maps/SpansTable.h @@ -4400,6 +4400,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6229,10 +6661,12 @@ template inline int64_t SpansTracksEntryMeasureBodyRetain( const template inline bool SpansTracksEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const SpansTracksEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SpansTracksEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const SpansTracksEntry & value, TableRetain * retain, const TableRetainPath & path ); inline bool SpansTracksEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, SpansTracksEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SpansTracksEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, SpansTracksEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t SpansMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Spans & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SpansSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Spans & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SpansSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Spans & value, TableRetain * retain, const TableRetainPath & path ); inline bool SpansLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Spans & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SpansLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Spans & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t SpansTracksEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const SpansTracksEntry & value ) @@ -8546,6 +8980,103 @@ inline bool SpansTracksEntryLoadBodyRetain( TableReader & r, const TableNodeMap } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SpansTracksEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, SpansTracksEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + SpansTracksEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 6 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 255ull ) { decoded_wide = (int64_t) 255ull; report->clamped++; } + uint8_t decoded_v = (uint8_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 1 ); + TableListFill fill = TableListFillBegin( nodes, value.value, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + Item * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + if ( !ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, ( *slot ), retain, TableRetainStepInto( path, 1, (uint32_t) ( i ) ) ) ) { return false; } + } + TableListFillEnd( fill ); + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t SpansMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Spans & value, TableRetain * retain, const TableRetainPath & path ) { @@ -8782,8 +9313,166 @@ inline bool SpansLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Sp } } -// SpansNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SpansLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Spans & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + SpansReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x755332609c470fbdull: // tracks + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.tracks, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint8_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const SpansTracksEntryMessageKeyRead read = SpansTracksEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + SpansTracksEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !SpansTracksEntryLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// SpansNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the // index is the region directory's, which Load fills from the wire's framing // and nothing afterwards renumbers. @@ -8955,6 +9644,182 @@ inline const Spans * SpansLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// SpansNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool SpansNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + SpansNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !SpansNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// SpansLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool SpansLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Spans * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !SpansMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = SpansNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + SpansNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !SpansMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Spans ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x033de3f1246bba76ull; + Spans * root = new ( region + used ) Spans; // lifetime only: LoadMessageBody's first act is SpansReset + SpansReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Spans ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !SpansNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return SpansLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// SpansLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool SpansLoadRetainMessages( const Spans ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !SpansLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // SpansMeasureRetain and SpansSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/TextTable.h b/testdata/golden/tables/maps/TextTable.h index 99a0cf947..790e3c2bf 100644 --- a/testdata/golden/tables/maps/TextTable.h +++ b/testdata/golden/tables/maps/TextTable.h @@ -4399,6 +4399,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6565,16 +6997,20 @@ inline TextBlobsEntryKeyRead TextBlobsEntryReadKey( const uint8_t * body, int64_ inline int64_t TextNamesEntryMeasureBodyRetain( TableRetainIds & ids, const TextNamesEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool TextNamesEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const TextNamesEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool TextNamesEntryLoadBodyRetain( TableReader & r, TextNamesEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TextNamesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, TextNamesEntry & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t TextWideEntryMeasureBodyRetain( TableRetainIds & ids, const TextWideEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool TextWideEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const TextWideEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool TextWideEntryLoadBodyRetain( TableReader & r, TextWideEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TextWideEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, TextWideEntry & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t TextBlobsEntryMeasureBodyRetain( TableRetainIds & ids, const TextBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool TextBlobsEntrySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const TextBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ); MAPDEMO_TABLE_INLINE bool TextBlobsEntryLoadBodyRetain( TableReader & r, TextBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TextBlobsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, TextBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t TextMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Text & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TextSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Text & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TextSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Text & value, TableRetain * retain, const TableRetainPath & path ); inline bool TextLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Text & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TextLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Text & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t TextNamesEntryMeasureBody( TableIds & ids, const TextNamesEntry & value ) { @@ -9800,6 +10236,98 @@ MAPDEMO_TABLE_INLINE bool TextNamesEntryLoadBodyRetain( TableReader & r, TextNam } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TextNamesEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, TextNamesEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + TextNamesEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); + value.key[kept] = 0; + r.offset += (int64_t) n * 8; + value.key_length = kept; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 16 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.value, text, (size_t) kept ); + value.value[kept] = 0; + r.offset += (int64_t) n * 8; + value.value_length = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + inline int64_t TextWideEntryMeasureBodyRetain( TableRetainIds & ids, const TextWideEntry & value, TableRetain * retain, const TableRetainPath & path ) { int64_t bytes = 1; // the ZERO REFERENCE that ends the body @@ -9923,7 +10451,127 @@ MAPDEMO_TABLE_INLINE bool TextWideEntryLoadBodyRetain( TableReader & r, TextWide } } -inline int64_t TextBlobsEntryMeasureBodyRetain( TableRetainIds & ids, const TextBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TextWideEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, TextWideEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + TextWideEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 7 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 7 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 65535ull ) { decoded_wide = (int64_t) 65535ull; report->clamped++; } + uint16_t decoded_v = (uint16_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 65535ull ) { decoded_wide = (int64_t) 65535ull; report->clamped++; } + uint16_t decoded_v = (uint16_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 33 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.has( (int64_t) n * 16 ) ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 6 ) { kept = 6; report->clamped++; } else { kept = (int32_t) n; } + bool high = false; // a high surrogate awaiting its low half + bool ill = false; + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t unit = 0; + if ( !r.get( unit, 16 ) ) { report->malformed = true; return false; } + const uint16_t u = (uint16_t) unit; + if ( u == 0 ) { ill = true; } + if ( high ) { if ( u < 0xDC00 || u > 0xDFFF ) { ill = true; } high = false; } + else if ( u >= 0xD800 && u <= 0xDBFF ) { high = true; } + else if ( u >= 0xDC00 && u <= 0xDFFF ) { ill = true; } + if ( (int32_t) i < kept ) { value.value[i] = (char16_t) u; } + } + if ( high || ill ) { report->malformed = true; return false; } + if ( kept > 0 && (uint64_t) kept < n ) + { + const char16_t last = value.value[kept - 1]; + if ( last >= 0xD800 && last <= 0xDBFF ) { kept--; } + } + value.value[kept] = 0; + value.value_length = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t TextBlobsEntryMeasureBodyRetain( TableRetainIds & ids, const TextBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ) { int64_t bytes = 1; // the ZERO REFERENCE that ends the body if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key @@ -10073,6 +10721,124 @@ MAPDEMO_TABLE_INLINE bool TextBlobsEntryLoadBodyRetain( TableReader & r, TextBlo } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TextBlobsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, TextBlobsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + TextBlobsEntryReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 6 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 10 ) { kept = 10; report->clamped++; } else { kept = (int32_t) n; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !r.get( by, 8 ) ) { report->malformed = true; return false; } + if ( (int32_t) i < kept ) { value.value[i] = (uint8_t) by; } + } + value.value_length = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t TextMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Text & value, TableRetain * retain, const TableRetainPath & path ) { @@ -10575,40 +11341,328 @@ inline bool TextLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Tex } } -// TextNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void TextNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TextLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Text & value, TableRetain * retain, const TableRetainPath & path ) { - // the node's own EXTENT, where its lists' and maps' arrays are carved - // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). - // The tool's path carries a worker instead: there the arrays are the - // arena's. - TableExtentCarve carve; - carve.worker = nodes.worker; - if ( carve.worker == NULL ) - { - TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse - const int64_t storage = TextNodeStorage( type_id, r.size, reason ); - const int64_t record = storage > 0 ? TextNodeRecordBytes( type_id ) : 0; - carve.at = at + record; - carve.left = storage > record ? storage - record : 0; - } - nodes.carve = &carve; - (void) nodes; // every node this root can name is a FIXED table - (void) retain; (void) node; - switch ( type_id ) + (void) nodes; (void) index_bits; + TextReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - default: break; - } - nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done -} - -// TextLoadRetain: decode the tolerant wire into the caller's exact-sized region and -// return the root. LOAD IS A SCAN, and that is the whole of its bound: it -// follows no reference, so there is no depth cap, no visited set and no + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xafb728fff268814full: // names + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.names, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + const char * last_key = NULL; int32_t last_length = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const TextNamesEntryMessageKeyRead read = TextNamesEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( last_key, last_length, read.key, read.length ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + TextNamesEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !TextNamesEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; last_length = read.length; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xa633f1f655715ccaull: // wide + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 1 ); + TableMapFill fill = TableMapFillBegin( nodes, value.wide, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint16_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const TextWideEntryMessageKeyRead read = TextWideEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + TextWideEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !TextWideEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 1, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0x14e2eaab9cde925bull: // blobs + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 2 ); + TableMapFill fill = TableMapFillBegin( nodes, value.blobs, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + int32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const TextBlobsEntryMessageKeyRead read = TextBlobsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (int64_t) last_key, (int64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + TextBlobsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !TextBlobsEntryLoadMessageBodyRetain( r, vocabulary, report, index_bits, *slot, retain, TableRetainStepInto( path, 2, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// TextNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void TextNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = TextNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? TextNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + (void) retain; (void) node; + switch ( type_id ) + { + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// TextLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no // ordering rule on the indices. Partial results are kept, as everywhere on // this wire — the report says what happened. NULL means the CALLER's buffer // was wrong. @@ -10748,6 +11802,182 @@ inline const Text * TextLoadRetain( uint8_t * region, int64_t region_bytes, cons return root; } +// TextNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool TextNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + TextNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !TextNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + (void) retain; (void) node; + bool ok = false; + switch ( type_id ) + { + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// TextLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool TextLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Text * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TextMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = TextNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + TextNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !TextMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Text ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x2492f5fb1b05b45eull; + Text * root = new ( region + used ) Text; // lifetime only: LoadMessageBody's first act is TextReset + TextReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Text ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !TextNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return TextLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// TextLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool TextLoadRetainMessages( const Text ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !TextLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // TextMeasureRetain and TextSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/maps/TrailsTable.h b/testdata/golden/tables/maps/TrailsTable.h index 6ad9b9906..c7a315d5b 100644 --- a/testdata/golden/tables/maps/TrailsTable.h +++ b/testdata/golden/tables/maps/TrailsTable.h @@ -4400,6 +4400,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace mapdemo #endif // MAPDEMO_SCHEMA_TABLE_RETAIN @@ -6239,10 +6671,12 @@ template inline int64_t TrailsStepsEntryMeasureBodyRetain( const template inline bool TrailsStepsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TrailsStepsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); inline bool TrailsStepsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TrailsStepsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t TrailsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TrailsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TrailsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); inline bool TrailsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Trails & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TrailsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Trails & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t TrailsStepsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const TrailsStepsEntry & value ) @@ -8717,92 +9151,205 @@ inline bool TrailsStepsEntryLoadBodyRetain( TableReader & r, const TableNodeMap } } -template -inline int64_t TrailsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - { - // steps: a kind 14 array of kind 13 elements, ASCENDING (§2.8) - TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); - if ( !order_steps.ok ) { return -1; } // the sort could not run - if ( order_steps.count > 0 ) - { - const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); - int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); // the element kind byte and the count - for ( int32_t i = 0; i < order_steps.count; i++ ) - { - const int64_t elem_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return -1; } - body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key - } - bytes += TableLebBytes( ref_steps ) + 1 + TableLebBytes( (uint64_t) ( body_steps ) ) + ( body_steps ); - } - TableMapRelease( order_steps ); - } - if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -template -inline bool TrailsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) -{ - { - TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps - if ( !order_steps.ok ) { return false; } - if ( order_steps.count > 0 ) // an EMPTY map elides, the by-value rule (§3) - { - const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); - int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); - for ( int32_t i = 0; i < order_steps.count; i++ ) - { - const int64_t elem_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return false; } - body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); - } - w.putleb( ref_steps ); w.put8( 14 ); w.putleb( (uint64_t) body_steps ); - w.put8( 13 ); w.putleb( (uint64_t) order_steps.count ); - for ( int32_t i = 0; i < order_steps.count; i++ ) - { - const int64_t elem_len_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); - if ( elem_len_steps < 0 ) { TableMapRelease( order_steps ); return false; } - w.putleb( (uint64_t) elem_len_steps ); - if ( !TrailsStepsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_steps ); return false; } - } - } - TableMapRelease( order_steps ); - } - if ( value.after != 0 ) - { - w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after - w.put32( uint32_t( value.after ) ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - return !w.overflow; -} - -template -inline bool TrailsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( !TrailsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -inline bool TrailsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Trails & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TrailsStepsEntryLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) { - TrailsReset( value ); // prefill declared defaults in place, then overlay + (void) nodes; (void) index_bits; + TrailsStepsEntryReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.value, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( *slot ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + TableListFillEnd( fill ); + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t TrailsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // steps: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); + if ( !order_steps.ok ) { return -1; } // the sort could not run + if ( order_steps.count > 0 ) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return -1; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_steps ) + 1 + TableLebBytes( (uint64_t) ( body_steps ) ) + ( body_steps ); + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool TrailsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return false; } + if ( order_steps.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return false; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); + } + w.putleb( ref_steps ); w.put8( 14 ); w.putleb( (uint64_t) body_steps ); + w.put8( 13 ); w.putleb( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_len_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_steps < 0 ) { TableMapRelease( order_steps ); return false; } + w.putleb( (uint64_t) elem_len_steps ); + if ( !TrailsStepsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_steps ); return false; } + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool TrailsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !TrailsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + TrailsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count const uint64_t field_id = r.ids->at( field_ref ); if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } uint8_t kind = r.get8(); @@ -8950,6 +9497,164 @@ inline bool TrailsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, T } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TrailsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + TrailsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x124250ad5a5b6d14ull: // steps + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.steps, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const TrailsStepsEntryMessageKeyRead read = TrailsStepsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + TrailsStepsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !TrailsStepsEntryLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // TrailsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -9123,6 +9828,182 @@ inline const Trails * TrailsLoadRetain( uint8_t * region, int64_t region_bytes, return root; } +// TrailsNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool TrailsNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + TableExtentCarve carve; + carve.at = at + TrailsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !TrailsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// TrailsLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool TrailsLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Trails * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TrailsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = TrailsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + TrailsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !TrailsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xb4578774a78fb150ull; + Trails * root = new ( region + used ) Trails; // lifetime only: LoadMessageBody's first act is TrailsReset + TrailsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Trails ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !TrailsNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return TrailsLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// TrailsLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool TrailsLoadRetainMessages( const Trails ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !TrailsLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // TrailsMeasureRetain and TrailsSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/pointers/GraphTable.h b/testdata/golden/tables/pointers/GraphTable.h index 2dcb91aac..bc454bf0e 100644 --- a/testdata/golden/tables/pointers/GraphTable.h +++ b/testdata/golden/tables/pointers/GraphTable.h @@ -4310,6 +4310,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace graphdemo #endif // GRAPHDEMO_SCHEMA_TABLE_RETAIN @@ -5105,33 +5537,41 @@ template inline bool TableNodeMessageSave( const Ctx & ctx, const inline int64_t MetaMeasureBodyRetain( TableRetainIds & ids, const Meta & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool MetaSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Meta & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool MetaLoadBodyRetain( TableReader & r, Meta & value, TableRetain * retain, const TableRetainPath & path ); +inline bool MetaLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Meta & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t SettingsMeasureBodyRetain( TableRetainIds & ids, const Settings & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool SettingsSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Settings & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool SettingsLoadBodyRetain( TableReader & r, Settings & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SettingsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Settings & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t ListNodeMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const ListNode & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ListNodeSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const ListNode & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ListNodeSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const ListNode & value, TableRetain * retain, const TableRetainPath & path ); inline bool ListNodeLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, ListNode & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ListNodeLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, ListNode & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t TreeNodeMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const TreeNode & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TreeNodeSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TreeNode & value, TableRetain * retain, const TableRetainPath & path ); template inline bool TreeNodeSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TreeNode & value, TableRetain * retain, const TableRetainPath & path ); inline bool TreeNodeLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, TreeNode & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TreeNodeLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, TreeNode & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t LayerMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Layer & value, TableRetain * retain, const TableRetainPath & path ); template inline bool LayerSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Layer & value, TableRetain * retain, const TableRetainPath & path ); template inline bool LayerSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Layer & value, TableRetain * retain, const TableRetainPath & path ); inline bool LayerLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Layer & value, TableRetain * retain, const TableRetainPath & path ); +inline bool LayerLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Layer & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t SceneMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Scene & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SceneSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Scene & value, TableRetain * retain, const TableRetainPath & path ); template inline bool SceneSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Scene & value, TableRetain * retain, const TableRetainPath & path ); inline bool SceneLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Scene & value, TableRetain * retain, const TableRetainPath & path ); +inline bool SceneLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Scene & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t DepotMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Depot & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DepotSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Depot & value, TableRetain * retain, const TableRetainPath & path ); template inline bool DepotSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Depot & value, TableRetain * retain, const TableRetainPath & path ); inline bool DepotLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Depot & value, TableRetain * retain, const TableRetainPath & path ); +inline bool DepotLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Depot & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t AlbumMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Album & value, TableRetain * retain, const TableRetainPath & path ); template inline bool AlbumSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Album & value, TableRetain * retain, const TableRetainPath & path ); template inline bool AlbumSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Album & value, TableRetain * retain, const TableRetainPath & path ); inline bool AlbumLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Album & value, TableRetain * retain, const TableRetainPath & path ); +inline bool AlbumLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Album & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t MetaMeasureBody( TableIds & ids, const Meta & value ) { @@ -15259,82 +15699,199 @@ GRAPHDEMO_TABLE_INLINE bool MetaLoadBodyRetain( TableReader & r, Meta & value, T } } -inline int64_t SettingsMeasureBodyRetain( TableRetainIds & ids, const Settings & value, TableRetain * retain, const TableRetainPath & path ) -{ - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.quality != 2 ) { bytes += TableLebBytes( ids.ref( 0x7a8060916400fe66ull ) ) + 1 + 4; } // quality - if ( value.label_length < 0 || value.label_length > 16 ) { return -1; } // storage invariant - if ( value.label_length > 0 ) { bytes += TableLebBytes( ids.ref( 0x39f7fcec8fcb623dull ) ) + 1 + TableLebBytes( (uint64_t) ( value.label_length ) ) + ( value.label_length ); } // label - bytes += TableRetainTailMeasure( retain, ids, path ); - return bytes; -} - -GRAPHDEMO_TABLE_INLINE bool SettingsSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Settings & value, TableRetain * retain, const TableRetainPath & path ) -{ - if ( value.quality != 2 ) - { - w.putleb( ids.ref( 0x7a8060916400fe66ull ) ); w.put8( 4 ); // quality - w.put32( uint32_t( value.quality ) ); - } - if ( value.label_length < 0 || value.label_length > 16 ) { return false; } // storage invariant - if ( value.label_length > 0 ) - { - w.putleb( ids.ref( 0x39f7fcec8fcb623dull ) ); w.put8( 12 ); // label - w.putleb( (uint64_t) value.label_length ); - w.raw( value.label, value.label_length ); - } - if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } - w.put8( 0 ); // the ZERO REFERENCE that ends the body - return !w.overflow; -} - -GRAPHDEMO_TABLE_INLINE bool SettingsLoadBodyRetain( TableReader & r, Settings & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool MetaLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Meta & value, TableRetain * retain, const TableRetainPath & path ) { - SettingsReset( value ); // prefill declared defaults in place, then overlay + MetaReset( value ); // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT - // (docs/SPEC-TABLES.md §6.6): this body is being established, so - // whatever an earlier occurrence of it left is discarded before the - // winning one is read. The discard moves neither counter. + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. TableRetainDiscardBody( retain, path ); for ( ;; ) { - uint64_t field_ref = 0; - if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } - if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE - if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count - const uint64_t field_id = r.ids->at( field_ref ); - if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } - uint8_t kind = r.get8(); - if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) - { - // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, - // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node - // table's is the ROOT body's alone, on the numbering's own - // rule — a second numbering cannot exist — and the BUILD - // VERSION's rides in the announcement and nowhere else. That - // body stops and the parent reads on past its L. - r.report->malformed = true; - return false; - } - switch ( field_id ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - case 0x7a8060916400fe66ull: // quality + case 0x802517e298c70b03ull: // build { - if ( kind != 4 ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) { - if ( TableKindWidens( kind, 4 ) ) + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) { - // WIDENED (§4): a kind that grew since the writer decodes - // exactly at its own width, the value lands, one widened counts - int64_t widened_v = 0; - if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } - int32_t decoded_v = (int32_t) widened_v; - if ( decoded_v < 0 ) { decoded_v = 0; r.report->clamped++; } - else if ( decoded_v > 4 ) { decoded_v = 4; r.report->clamped++; } - value.quality = decoded_v; - r.report->widened++; - break; - } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 1000ll ) { decoded_wide = 1000ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.build = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 1000ll ) { decoded_wide = 1000ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.build = decoded_v; + } + break; + } + case 0x56d7ab194448a4f3ull: // tag + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.tag, text, (size_t) kept ); + value.tag[kept] = 0; + r.offset += (int64_t) n * 8; + value.tag_length = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +inline int64_t SettingsMeasureBodyRetain( TableRetainIds & ids, const Settings & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.quality != 2 ) { bytes += TableLebBytes( ids.ref( 0x7a8060916400fe66ull ) ) + 1 + 4; } // quality + if ( value.label_length < 0 || value.label_length > 16 ) { return -1; } // storage invariant + if ( value.label_length > 0 ) { bytes += TableLebBytes( ids.ref( 0x39f7fcec8fcb623dull ) ) + 1 + TableLebBytes( (uint64_t) ( value.label_length ) ) + ( value.label_length ); } // label + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +GRAPHDEMO_TABLE_INLINE bool SettingsSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Settings & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.quality != 2 ) + { + w.putleb( ids.ref( 0x7a8060916400fe66ull ) ); w.put8( 4 ); // quality + w.put32( uint32_t( value.quality ) ); + } + if ( value.label_length < 0 || value.label_length > 16 ) { return false; } // storage invariant + if ( value.label_length > 0 ) + { + w.putleb( ids.ref( 0x39f7fcec8fcb623dull ) ); w.put8( 12 ); // label + w.putleb( (uint64_t) value.label_length ); + w.raw( value.label, value.label_length ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +GRAPHDEMO_TABLE_INLINE bool SettingsLoadBodyRetain( TableReader & r, Settings & value, TableRetain * retain, const TableRetainPath & path ) +{ + SettingsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x7a8060916400fe66ull: // quality + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + if ( decoded_v < 0 ) { decoded_v = 0; r.report->clamped++; } + else if ( decoded_v > 4 ) { decoded_v = 4; r.report->clamped++; } + value.quality = decoded_v; + r.report->widened++; + break; + } // AT A POSITION THE READER DOES NAME, a field under // kind 31 or kind 32 takes this same rule and no other (§3) r.report->kind_mismatch++; @@ -15381,6 +15938,123 @@ GRAPHDEMO_TABLE_INLINE bool SettingsLoadBodyRetain( TableReader & r, Settings & } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SettingsLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Settings & value, TableRetain * retain, const TableRetainPath & path ) +{ + SettingsReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x7a8060916400fe66ull: // quality + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 4ll ) { decoded_wide = 4ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.quality = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 4ll ) { decoded_wide = 4ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.quality = decoded_v; + } + break; + } + case 0x39f7fcec8fcb623dull: // label + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 16 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.label, text, (size_t) kept ); + value.label[kept] = 0; + r.offset += (int64_t) n * 8; + value.label_length = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t ListNodeMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const ListNode & value, TableRetain * retain, const TableRetainPath & path ) { @@ -15503,56 +16177,192 @@ inline bool ListNodeLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, { if ( kind != 12 ) { - // AT A POSITION THE READER DOES NAME, a field under - // kind 31 or kind 32 takes this same rule and no other (§3) - r.report->kind_mismatch++; - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t len = 0; + if ( !r.getleb( len ) || !r.room( len ) ) { r.report->malformed = true; return false; } + // ILL-FORMED TEXT IS DAMAGE (§3, §4): the field reads its declared + // default, one malformed counts, and the parent reads on past L + if ( !TableUtf8Valid( r.buffer + r.offset, len ) ) { r.report->malformed = true; value.name[0] = 0; value.name_length = 0; r.offset += (int64_t) len; break; } + uint64_t keep = len; + if ( keep > 12 ) { keep = (uint64_t) TableUtf8Clamp( r.buffer + r.offset, len, 12 ); r.report->clamped++; } // at a code point boundary (§3) + memcpy( value.name, r.buffer + r.offset, (size_t) keep ); + value.name[keep] = 0; + value.name_length = (int32_t) keep; + r.offset += (int64_t) len; + break; + } + case 0xe5316cbaa025f028ull: // next + { + if ( kind != 17 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + // A POINTER FIELD'S PAYLOAD IS A NUMBER (docs/SPEC-TABLES.md §3.1): it is + // bounds-checked and resolved through the numbering, never FOLLOWED, so + // there is no traversal here and therefore no traversal bound. + { + uint64_t node_index = 0; + if ( !r.getleb( node_index ) ) { r.report->malformed = true; return false; } + TableNodeResolve( nodes, value.next, node_index, 0xf60ec899a5a69fa9ull, r.report ); // *ListNode + } + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ListNodeLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, ListNode & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + ListNodeReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.value = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.value = decoded_v; + } + break; + } + case 0xc4bcadba8e631b86ull: // name + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } break; } - uint64_t len = 0; - if ( !r.getleb( len ) || !r.room( len ) ) { r.report->malformed = true; return false; } - // ILL-FORMED TEXT IS DAMAGE (§3, §4): the field reads its declared - // default, one malformed counts, and the parent reads on past L - if ( !TableUtf8Valid( r.buffer + r.offset, len ) ) { r.report->malformed = true; value.name[0] = 0; value.name_length = 0; r.offset += (int64_t) len; break; } - uint64_t keep = len; - if ( keep > 12 ) { keep = (uint64_t) TableUtf8Clamp( r.buffer + r.offset, len, 12 ); r.report->clamped++; } // at a code point boundary (§3) - memcpy( value.name, r.buffer + r.offset, (size_t) keep ); - value.name[keep] = 0; - value.name_length = (int32_t) keep; - r.offset += (int64_t) len; + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 12 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.name, text, (size_t) kept ); + value.name[kept] = 0; + r.offset += (int64_t) n * 8; + value.name_length = kept; + } break; } case 0xe5316cbaa025f028ull: // next { - if ( kind != 17 ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) { - // AT A POSITION THE READER DOES NAME, a field under - // kind 31 or kind 32 takes this same rule and no other (§3) - r.report->kind_mismatch++; - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } break; } - // A POINTER FIELD'S PAYLOAD IS A NUMBER (docs/SPEC-TABLES.md §3.1): it is - // bounds-checked and resolved through the numbering, never FOLLOWED, so - // there is no traversal here and therefore no traversal bound. { uint64_t node_index = 0; - if ( !r.getleb( node_index ) ) { r.report->malformed = true; return false; } - TableNodeResolve( nodes, value.next, node_index, 0xf60ec899a5a69fa9ull, r.report ); // *ListNode + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.next, node_index, 0xf60ec899a5a69fa9ull, report ); // *ListNode } break; } - case 0xffffffffffffffffull: - { - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } - break; - } default: + report->unknown++; { - r.report->unknown++; - if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } - break; + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); } + break; } } } @@ -15745,6 +16555,110 @@ inline bool TreeNodeLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TreeNodeLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, TreeNode & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + TreeNodeReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x39f7fcec8fcb623dull: // label + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 12 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.label, text, (size_t) kept ); + value.label[kept] = 0; + r.offset += (int64_t) n * 8; + value.label_length = kept; + } + break; + } + case 0x24b070ada2041cb0ull: // left + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.left, node_index, 0xb97e90a3784c431dull, report ); // *TreeNode + } + break; + } + case 0x76aaaa535714d805ull: // right + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.right, node_index, 0xb97e90a3784c431dull, report ); // *TreeNode + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t LayerMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Layer & value, TableRetain * retain, const TableRetainPath & path ) { @@ -15873,22 +16787,133 @@ inline bool LayerLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, La // there is no traversal here and therefore no traversal bound. { uint64_t node_index = 0; - if ( !r.getleb( node_index ) ) { r.report->malformed = true; return false; } - TableNodeResolve( nodes, value.head, node_index, 0xf60ec899a5a69fa9ull, r.report ); // *ListNode + if ( !r.getleb( node_index ) ) { r.report->malformed = true; return false; } + TableNodeResolve( nodes, value.head, node_index, 0xf60ec899a5a69fa9ull, r.report ); // *ListNode + } + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool LayerLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Layer & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + LayerReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x75d8e97600b296eaull: // depth + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 64ll ) { decoded_wide = 64ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.depth = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 64ll ) { decoded_wide = 64ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.depth = decoded_v; + } + break; + } + case 0x0a8f12cc5f9a0c03ull: // head + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.head, node_index, 0xf60ec899a5a69fa9ull, report ); // *ListNode } break; } - case 0xffffffffffffffffull: - { - if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } - break; - } default: + report->unknown++; { - r.report->unknown++; - if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } - break; + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); } + break; } } } @@ -16389,6 +17414,261 @@ inline bool SceneLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Sc } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool SceneLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Scene & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + SceneReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xc4bcadba8e631b86ull: // name + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 24 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.name, text, (size_t) kept ); + value.name[kept] = 0; + r.offset += (int64_t) n * 8; + value.name_length = kept; + } + break; + } + case 0xbb62c62c9808ea37ull: // version + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 99ll ) { decoded_wide = 99ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.version = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 99ll ) { decoded_wide = 99ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.version = decoded_v; + } + break; + } + case 0x0a8f12cc5f9a0c03ull: // head + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.head, node_index, 0xf60ec899a5a69fa9ull, report ); // *ListNode + } + break; + } + case 0x5b25b8ef511eb395ull: // tree + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.tree, node_index, 0xb97e90a3784c431dull, report ); // *TreeNode + } + break; + } + case 0xee5f6d7b48b44de8ull: // settings + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.settings, node_index, 0x9d8b8aa2b404c2c8ull, report ); // *Settings + } + break; + } + case 0x509220bb65a646b7ull: // alias + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.alias, node_index, 0xf60ec899a5a69fa9ull, report ); // *ListNode + } + break; + } + case 0x60839e2395be697eull: // ground + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 6 ); + if ( !LayerLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, value.ground, retain, TableRetainStepInto( path, 6, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0x4554e34a747022dfull: // layers + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 4 ) { kept = 4; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + Layer scratch; + LayerReset( scratch ); + if ( !LayerLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( in_bounds ? value.layers[i] : scratch ), ( in_bounds ? retain : NULL ), TableRetainStepInto( path, 7, (uint32_t) ( i ) ) ) ) { return false; } + } + value.layers_count = kept; + } + break; + } + case 0x4320e9a2e32eac38ull: // meta + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 8 ); + if ( !MetaLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.meta, retain, TableRetainStepInto( path, 8, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t DepotMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Depot & value, TableRetain * retain, const TableRetainPath & path ) { @@ -16688,6 +17968,140 @@ inline bool DepotLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, De } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool DepotLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Depot & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + DepotReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xc4bcadba8e631b86ull: // name + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 12 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.name, text, (size_t) kept ); + value.name[kept] = 0; + r.offset += (int64_t) n * 8; + value.name_length = kept; + } + break; + } + case 0xdd9eb981e152e90cull: // banks + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 16 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { report->malformed = true; return false; } + for ( uint64_t p = 0; p < n; p++ ) + { + uint64_t key_ref = 0; + if ( !r.get( key_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry key_entry; + if ( !TableMessageNameEntry( vocabulary, key_ref, key_entry ) ) { report->malformed = true; return false; } + Tier key = Tier::None; + const bool named = TableEnumValue( key_entry.id, key ); + if ( !named ) { report->unknown++; report->retain_lost++; } + const int32_t slot = named ? (int32_t) key - 1 : -1; + const bool in_bounds = slot >= 0 && slot < 2; + Layer scratch; + LayerReset( scratch ); + if ( !LayerLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, ( in_bounds ? value.banks.slots[slot] : scratch ), ( in_bounds ? retain : NULL ), TableRetainStepInto( path, 1, (uint32_t) ( slot ) ) ) ) { return false; } + } + } + break; + } + case 0x4339ee8ab21c8380ull: // spare + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 2 ); + if ( !MetaLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.spare, retain, TableRetainStepInto( path, 2, (uint32_t) ( 0 ) ) ) ) { return false; } + value.spare_present = true; // the field rode, so it is PRESENT (§2.3) + break; + } + case 0x0a8f12cc5f9a0c03ull: // head + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.head, node_index, 0xf60ec899a5a69fa9ull, report ); // *ListNode + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t AlbumMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Album & value, TableRetain * retain, const TableRetainPath & path ) { @@ -17011,6 +18425,155 @@ inline bool AlbumLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Al } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool AlbumLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Album & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + AlbumReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xc4bcadba8e631b86ull: // name + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 16 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.name, text, (size_t) kept ); + value.name[kept] = 0; + r.offset += (int64_t) n * 8; + value.name_length = kept; + } + break; + } + case 0x1e4984ef2e958a4cull: // tint + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + if ( !ColourLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.tint, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0xee7ba9ad45c64144ull: // stamp + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 2 ); + if ( !StampLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.stamp, retain, TableRetainStepInto( path, 2, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0xeddcb72b15486e77ull: // marker + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 13 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 3 ); + if ( !MarkerLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, value.marker, retain, TableRetainStepInto( path, 3, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0x77af761956600b54ull: // pin + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.pin, node_index, 0xd6458a3eef83d457ull, report ); // *Marker + } + break; + } + case 0x0a8f12cc5f9a0c03ull: // head + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.head, node_index, 0xf60ec899a5a69fa9ull, report ); // *ListNode + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // ListNodeNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -17161,6 +18724,160 @@ inline const ListNode * ListNodeLoadRetain( uint8_t * region, int64_t region_byt return root; } +// ListNodeNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool ListNodeNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0xf60ec899a5a69fa9ull: ok = ListNodeLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(ListNode *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // ListNode + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// ListNodeLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool ListNodeLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const ListNode * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !ListNodeMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = ListNodeNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + ListNodeNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( ListNode ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xf60ec899a5a69fa9ull; + ListNode * root = new ( region + used ) ListNode; // lifetime only: LoadMessageBody's first act is ListNodeReset + ListNodeReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !ListNodeNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return ListNodeLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// ListNodeLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool ListNodeLoadRetainMessages( const ListNode ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !ListNodeLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // ListNodeMeasureRetain and ListNodeSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save @@ -17420,13 +19137,167 @@ inline const TreeNode * TreeNodeLoadRetain( uint8_t * region, int64_t region_byt k++; } } - - // and the ROOT's own body last, so every index it carries resolves - // against a numbering already known good or already known bad - TableReader r( wire, wire_bytes, out, &ids_table ); - r.nested = false; // the ROOT body, the one that carries the node table - TreeNodeLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); - return root; + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TreeNodeLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// TreeNodeNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool TreeNodeNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0xb97e90a3784c431dull: ok = TreeNodeLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(TreeNode *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // TreeNode + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// TreeNodeLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool TreeNodeLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const TreeNode * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TreeNodeMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = TreeNodeNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + TreeNodeNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( TreeNode ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xb97e90a3784c431dull; + TreeNode * root = new ( region + used ) TreeNode; // lifetime only: LoadMessageBody's first act is TreeNodeReset + TreeNodeReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !TreeNodeNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return TreeNodeLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// TreeNodeLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool TreeNodeLoadRetainMessages( const TreeNode ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !TreeNodeLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); } // TreeNodeMeasureRetain and TreeNodeSaveRetain: the pair, with the retained tail in @@ -17697,6 +19568,160 @@ inline const Layer * LayerLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// LayerNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool LayerNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0xf60ec899a5a69fa9ull: ok = ListNodeLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(ListNode *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // ListNode + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// LayerLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool LayerLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Layer * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !LayerMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = LayerNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + LayerNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Layer ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x9d167ef77aed79b6ull; + Layer * root = new ( region + used ) Layer; // lifetime only: LoadMessageBody's first act is LayerReset + LayerReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !LayerNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return LayerLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// LayerLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool LayerLoadRetainMessages( const Layer ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !LayerLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // LayerMeasureRetain and LayerSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save @@ -17967,6 +19992,162 @@ inline const Scene * SceneLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// SceneNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool SceneNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0xf60ec899a5a69fa9ull: ok = ListNodeLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(ListNode *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // ListNode + case 0xb97e90a3784c431dull: ok = TreeNodeLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(TreeNode *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // TreeNode + case 0x9d8b8aa2b404c2c8ull: ok = SettingsLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Settings *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Settings + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// SceneLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool SceneLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Scene * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !SceneMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = SceneNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + SceneNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Scene ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x4a9a31623ab5f213ull; + Scene * root = new ( region + used ) Scene; // lifetime only: LoadMessageBody's first act is SceneReset + SceneReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !SceneNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return SceneLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// SceneLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool SceneLoadRetainMessages( const Scene ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !SceneLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // SceneMeasureRetain and SceneSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save @@ -18241,6 +20422,160 @@ inline const Depot * DepotLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// DepotNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool DepotNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0xf60ec899a5a69fa9ull: ok = ListNodeLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(ListNode *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // ListNode + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// DepotLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool DepotLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Depot * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !DepotMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = DepotNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + DepotNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Depot ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x327fe6dc702553fdull; + Depot * root = new ( region + used ) Depot; // lifetime only: LoadMessageBody's first act is DepotReset + DepotReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !DepotNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return DepotLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// DepotLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool DepotLoadRetainMessages( const Depot ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !DepotLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // DepotMeasureRetain and DepotSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save @@ -18511,6 +20846,162 @@ inline const Album * AlbumLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// AlbumNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool AlbumNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0x69ff34904242a73dull: ok = TallyLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Tally *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Tally + case 0xd6458a3eef83d457ull: ok = MarkerLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(Marker *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Marker + case 0xf60ec899a5a69fa9ull: ok = ListNodeLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(ListNode *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // ListNode + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// AlbumLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool AlbumLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Album * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !AlbumMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = AlbumNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + AlbumNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Album ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xd858c2cb7f1514ccull; + Album * root = new ( region + used ) Album; // lifetime only: LoadMessageBody's first act is AlbumReset + AlbumReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !AlbumNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return AlbumLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// AlbumLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool AlbumLoadRetainMessages( const Album ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !AlbumLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // AlbumMeasureRetain and AlbumSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/pointers/MarksTable.h b/testdata/golden/tables/pointers/MarksTable.h index 1a53962ce..09fedc842 100644 --- a/testdata/golden/tables/pointers/MarksTable.h +++ b/testdata/golden/tables/pointers/MarksTable.h @@ -4307,6 +4307,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace graphdemo #endif // GRAPHDEMO_SCHEMA_TABLE_RETAIN @@ -4787,10 +5219,12 @@ template inline bool TableNodeMessageSave( const Ctx & ctx, const inline int64_t TallyMeasureBodyRetain( TableRetainIds & ids, const Tally & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool TallySaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Tally & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool TallyLoadBodyRetain( TableReader & r, Tally & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TallyLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Tally & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t MarkerMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Marker & value, TableRetain * retain, const TableRetainPath & path ); template inline bool MarkerSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Marker & value, TableRetain * retain, const TableRetainPath & path ); template inline bool MarkerSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Marker & value, TableRetain * retain, const TableRetainPath & path ); inline bool MarkerLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Marker & value, TableRetain * retain, const TableRetainPath & path ); +inline bool MarkerLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Marker & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t TallyMeasureBody( TableIds & ids, const Tally & value ) { @@ -6471,6 +6905,98 @@ GRAPHDEMO_TABLE_INLINE bool TallyLoadBodyRetain( TableReader & r, Tally & value, } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool TallyLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Tally & value, TableRetain * retain, const TableRetainPath & path ) +{ + TallyReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x732dfbcc9b0cf0bbull: // hits + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 10000ll ) { decoded_wide = 10000ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.hits = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 10000ll ) { decoded_wide = 10000ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.hits = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t MarkerMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Marker & value, TableRetain * retain, const TableRetainPath & path ) { @@ -6615,6 +7141,92 @@ inline bool MarkerLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, M } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool MarkerLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Marker & value, TableRetain * retain, const TableRetainPath & path ) +{ + (void) nodes; (void) index_bits; + MarkerReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x39f7fcec8fcb623dull: // label + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.label, text, (size_t) kept ); + value.label[kept] = 0; + r.offset += (int64_t) n * 8; + value.label_length = kept; + } + break; + } + case 0x3bf8fbbad1587cddull: // note + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.note, node_index, 0x69ff34904242a73dull, report ); // *Tally + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // MarkerNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it // already owns. // EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the @@ -6766,6 +7378,161 @@ inline const Marker * MarkerLoadRetain( uint8_t * region, int64_t region_bytes, return root; } +// MarkerNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool MarkerNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x69ff34904242a73dull: ok = TallyLoadMessageBodyRetain( r, vocabulary, report, index_bits, *(Tally *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Tally + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// MarkerLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool MarkerLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Marker * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !MarkerMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = MarkerNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + MarkerNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Marker ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xd6458a3eef83d457ull; + Marker * root = new ( region + used ) Marker; // lifetime only: LoadMessageBody's first act is MarkerReset + MarkerReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !MarkerNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return MarkerLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// MarkerLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool MarkerLoadRetainMessages( const Marker ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !MarkerLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // MarkerMeasureRetain and MarkerSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/golden/tables/pointers/PartsTable.h b/testdata/golden/tables/pointers/PartsTable.h index 6130a6aef..8642f4dd8 100644 --- a/testdata/golden/tables/pointers/PartsTable.h +++ b/testdata/golden/tables/pointers/PartsTable.h @@ -4307,6 +4307,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace graphdemo #endif // GRAPHDEMO_SCHEMA_TABLE_RETAIN @@ -4689,9 +5121,11 @@ GRAPHDEMO_TABLE_INLINE bool ColourLoadBody( TableReader & r, Colour & value ); inline int64_t StampMeasureBodyRetain( TableRetainIds & ids, const Stamp & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool StampSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Stamp & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool StampLoadBodyRetain( TableReader & r, Stamp & value, TableRetain * retain, const TableRetainPath & path ); +inline bool StampLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Stamp & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t ColourMeasureBodyRetain( TableRetainIds & ids, const Colour & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool ColourSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Colour & value, TableRetain * retain, const TableRetainPath & path ); GRAPHDEMO_TABLE_INLINE bool ColourLoadBodyRetain( TableReader & r, Colour & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ColourLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Colour & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t StampMeasureBody( TableIds & ids, const Stamp & value ) { @@ -5624,6 +6058,123 @@ GRAPHDEMO_TABLE_INLINE bool StampLoadBodyRetain( TableReader & r, Stamp & value, } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool StampLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Stamp & value, TableRetain * retain, const TableRetainPath & path ) +{ + StampReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x56d7ab194448a4f3ull: // tag + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.tag, text, (size_t) kept ); + value.tag[kept] = 0; + r.offset += (int64_t) n * 8; + value.tag_length = kept; + } + break; + } + case 0x823b8a195ce2133cull: // seq + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 1000ll ) { decoded_wide = 1000ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.seq = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < 0ll ) { decoded_wide = 0ll; report->clamped++; } + if ( decoded_wide > 1000ll ) { decoded_wide = 1000ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.seq = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + inline int64_t ColourMeasureBodyRetain( TableRetainIds & ids, const Colour & value, TableRetain * retain, const TableRetainPath & path ) { int64_t bytes = 1; // the ZERO REFERENCE that ends the body @@ -5741,6 +6292,117 @@ GRAPHDEMO_TABLE_INLINE bool ColourLoadBodyRetain( TableReader & r, Colour & valu } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ColourLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Colour & value, TableRetain * retain, const TableRetainPath & path ) +{ + ColourReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xaf63ef4c86020cd5ull: // r + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 6 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 255ull ) { decoded_wide = (int64_t) 255ull; report->clamped++; } + uint8_t decoded_v = (uint8_t) decoded_wide; + value.r = decoded_v; + } + break; + } + case 0xaf63da4c8601e926ull: // g + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 6 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 255ull ) { decoded_wide = (int64_t) 255ull; report->clamped++; } + uint8_t decoded_v = (uint8_t) decoded_wide; + value.g = decoded_v; + } + break; + } + case 0xaf63df4c8601f1a5ull: // b + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 6 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 255ull ) { decoded_wide = (int64_t) 255ull; report->clamped++; } + uint8_t decoded_v = (uint8_t) decoded_wide; + value.b = decoded_v; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + // ---- retain-unknown on a FIXED-class root: refused by name (§6.6) ---- // // A fixed-class root is a VALUE: no region, no node directory, and so no diff --git a/testdata/golden/tables/stream/StreamTable.h b/testdata/golden/tables/stream/StreamTable.h index 11f62058e..acbece913 100644 --- a/testdata/golden/tables/stream/StreamTable.h +++ b/testdata/golden/tables/stream/StreamTable.h @@ -4153,6 +4153,438 @@ inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRet } return true; } + +// ---- RETENTION ON THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3) ---- +// +// LoadRetain reads a form 2 body as it reads a file's, the resolving walk +// replacing every reference with the id it names AGAINST THE CONNECTION'S +// VOCABULARY instead of a trailer, and SaveRetain writing form 2 refuses by +// name. So retention crosses the forms in ONE DIRECTION, and the record a +// message body produces is the FILE FORM'S OWN, to the byte: a retained record +// carries the field's bytes with every reference resolved so that re-emitting +// it into any id table is correct, and the table it is re-emitted into is a +// file's. The capture below is therefore a TRANSCODE as well as a resolve, a +// bitpacked value is read at the width its announced shape states and written +// at the width the file form spells, and from there it is the same record, +// laid down in the same slots and read back by the same emit walk. +// +// THE SKIP RUNS FIRST AND THE CAPTURE SECOND, over the same bits. The plain +// read's verdict on an unknown entry is the SKIP's, whether or not this build +// retains, and running the skip first is what keeps that verdict exactly what +// it was with retention off. The capture re-reads the bits the skip delimited, +// and a capture that lands anywhere but the skip's own end drops the record. It +// is two flat passes over one record's bits and never a walk that doubles at +// every level, which is what the cost rule forbids. + +struct TableMessageRetainIn +{ + TableRetainIn out; // its in/size/at/ids are unused: the input is BITS + TableBitReader * r; + const TableVocabulary * vocabulary; + int64_t index_bits; +}; + +// THE ELEMENT'S OWN ENTRY, which a resolved entry carries flattened beside its +// own: an array's and a keyed body's elements are read through it. +inline TableMessageEntry TableMessageRetainElement( const TableMessageEntry & entry ) +{ + TableMessageEntry e; + e.kind = entry.elem_kind; + e.packing = entry.elem_packing; + e.value_bits = entry.elem_value_bits; + e.max = entry.elem_max; + e.base_lo = entry.elem_base_lo; + e.base_hi = entry.elem_base_hi; + e.qmin = entry.elem_qmin; + e.qdelta = entry.elem_qdelta; + e.qcount = entry.elem_qcount; + return e; +} + +// the kinds whose file bytes are TWO'S COMPLEMENT, which is what decides +// whether a narrow raw value sign-extends into its storage width +inline bool TableMessageRetainSigned( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 3: case 4: case 5: case 18: + case 20: case 21: case 22: case 23: case 24: return true; + default: return false; + } +} + +inline int64_t TableMessageRetainWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + default: return -1; + } +} + +// one integer at the FILE's own width for its kind, little endian +inline void TableMessageRetainWord( TableMessageRetainIn & s, uint64_t value, int64_t width ) +{ + uint8_t b[8]; + for ( int64_t i = 0; i < width; i++ ) { b[i] = uint8_t( value >> ( 8 * i ) ); } + TableRetainInRaw( s.out, b, width ); +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ); + +// ONE REFERENCE resolved against the announced vocabulary. It is the file +// walk's own verdict read against an announcement instead of a trailer: a +// reference of zero where an entry is required, one above the entry count, and +// one naming a RESERVED id, which would be re-emitted into a nested body where +// it is malformed, all DROP THE RECORD. +inline bool TableMessageRetainId( TableMessageRetainIn & s, uint64_t ref, uint64_t & id ) +{ + if ( ref == 0 || ref > (uint64_t) s.vocabulary->count ) { return false; } + const TableMessageEntry & named = TableVocabularyEntryAt( *s.vocabulary, ref ); + if ( named.id == 0 || TableRetainReservedId( named.id ) ) { return false; } + id = named.id; + return true; +} + +inline bool TableMessageRetainRef( TableMessageRetainIn & s, uint64_t & id ) +{ + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return false; } + return TableMessageRetainId( s, ref, id ); +} + +// ONE OPAQUE PAYLOAD's bytes: a string or a byte buffer is a length at its own +// width, the ALIGN that buys a memcpy and the bytes; a wide string is a length +// and then SIXTEEN bits a code unit, which is two file bytes each; and the +// ESCAPE aligns, reads a thirty-two bit L and takes L bytes. On the file each +// is the same bytes behind a canonical LEB128 length, so the framed flag says which of +// the two spellings this position takes. +inline bool TableMessageRetainOpaque( TableMessageRetainIn & s, const TableMessageEntry & entry, bool framed ) +{ + uint64_t n = 0; + if ( entry.kind == 12 || entry.kind == 33 ) + { + if ( !s.r->get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + if ( entry.kind == 33 ) { n *= 2; } + else if ( !s.r->align() ) { return false; } + } + else + { + if ( !s.r->align() || !s.r->get( n, 32 ) ) { return false; } + } + if ( n > (uint64_t) INT32_MAX || !s.r->has( (int64_t) n * 8 ) ) { return false; } + if ( framed ) { TableRetainInLeb( s.out, n ); } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !s.r->get( by, 8 ) ) { return false; } + const uint8_t one = uint8_t( by ); + TableRetainInRaw( s.out, &one, 1 ); + } + return true; +} + +// ONE ANNOUNCED VALUE, transcoded. NO CLAMP FIRES AND NO RANGE IS APPLIED: the +// value is the SENDER's, under an id this reader cannot name, so there is no +// declaration to bound it by, and the walk takes no branch on a payload byte +// (§6.6, THE SECURITY BOUND). +inline bool TableMessageRetainScalar( TableMessageRetainIn & s, const TableMessageEntry & entry ) +{ + const int64_t width = TableMessageRetainWidth( entry.kind ); + if ( width < 0 ) { return false; } + if ( entry.kind == 1 ) + { + uint64_t v = 0; + if ( !s.r->get( v, 1 ) ) { return false; } + TableMessageRetainWord( s, v, 1 ); + return true; + } + if ( entry.kind == 10 ) + { + if ( entry.packing == 2 ) + { + // THE PACKET WIRE'S RULE, IN FLOAT32: the float an index names is + // the float a packet's reader names for it, and an index above + // the announced count is rejected rather than reconstructed (SPEC.md §4.3) + uint64_t index = 0; + if ( !s.r->get( index, entry.value_bits ) ) { return false; } + if ( index > (uint64_t) entry.qcount ) { return false; } + const float v = TableMessageDequantize( (uint32_t) index, entry.qmin, entry.qdelta, entry.qcount ); + TableMessageRetainWord( s, table_float_to_bits( v ), 4 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, 32 ) ) { return false; } + TableMessageRetainWord( s, raw, 4 ); + return true; + } + if ( entry.kind == 11 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, 64 ) ) { return false; } + TableMessageRetainWord( s, raw, 8 ); + return true; + } + const int64_t bits = entry.value_bits; + if ( bits < 0 ) { return false; } + if ( width == 16 ) + { + // a 128-bit kind reads in TWO LIMBS, and a ranged one adds the base the + // announcement carries in halves + uint64_t lo = 0, hi = 0; + if ( bits <= 64 ) + { + if ( !s.r->get( lo, bits ) ) { return false; } + } + else if ( !s.r->get( lo, 64 ) || !s.r->get( hi, bits - 64 ) ) { return false; } + if ( entry.packing == 1 ) + { + const uint64_t sum = lo + (uint64_t) entry.base_lo; + hi += (uint64_t) entry.base_hi + ( sum < lo ? 1u : 0u ); + lo = sum; + } + TableMessageRetainWord( s, lo, 8 ); + TableMessageRetainWord( s, hi, 8 ); + return true; + } + uint64_t raw = 0; + if ( !s.r->get( raw, bits ) ) { return false; } + uint64_t value = raw; + if ( entry.packing == 1 ) + { + value = raw + (uint64_t) entry.base_lo; // the domain, whole: the sum wraps into the bits below + } + else if ( TableMessageRetainSigned( entry.kind ) && bits > 0 && bits < 64 ) + { + const int64_t shift = 64 - bits; + value = (uint64_t) ( ( (int64_t) ( raw << shift ) ) >> shift ); + } + TableMessageRetainWord( s, value, width ); + return true; +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it, +// exactly as the file capture's own framed content is. +inline int64_t TableMessageRetainFramed( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s.out ); + const int64_t resolved = TableMessageRetainContent( s, entry, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s.out, slot, resolved ); + return resolved; +} + +// one nested BODY: its fields, each an entry reference and the payload that +// entry's shape frames, ending at the body's own ZERO REFERENCE. +inline int64_t TableMessageRetainBody( TableMessageRetainIn & s, int32_t depth ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // THE TERMINATOR IS A REFERENCE, and a reference in the resolved + // form is a fixed eight-byte id + TableRetainInId( s.out, 0 ); + return 0; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + const TableMessageEntry & field = TableVocabularyEntryAt( *s.vocabulary, ref ); + TableRetainInId( s.out, id ); + TableRetainInRaw( s.out, &field.kind, 1 ); + if ( TableMessageRetainPayload( s, field, depth ) < 0 ) { return -1; } + } +} + +// one array's or one keyed body's content: the element kind, the count, and the +// elements. A KEYED SLOT IS A TRIPLE, the key reference, the element's own +// length, and the element, and its keys resolve at EVERY element kind. +inline int64_t TableMessageRetainElements( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( entry.elem_kind == 17 ) + { + // AN ARRAY WHOSE ELEMENT KIND IS 17 (§6.6): the excluded class, caught + // here for a keyed body exactly as for a positional one + return -1; + } + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !s.r->get( raw, width ) ) { return -1; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !s.r->align() ) { return -1; } + const TableMessageEntry element = TableMessageRetainElement( entry ); + TableRetainInRaw( s.out, &entry.elem_kind, 1 ); + TableRetainInLeb( s.out, n ); // the count rides as the file capture spells it, and the emit reads it back + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 ) + { + uint64_t key = 0; + if ( !TableMessageRetainRef( s, key ) ) { return -1; } + TableRetainInId( s.out, key ); + if ( TableMessageRetainFramed( s, element, depth ) < 0 ) { return -1; } + continue; + } + if ( TableMessageRetainPayload( s, element, depth ) < 0 ) { return -1; } + } + return 0; +} + +inline int64_t TableMessageRetainContent( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + case 13: if ( TableMessageRetainBody( s, depth ) < 0 ) { return -1; } break; + case 14: case 16: if ( TableMessageRetainElements( s, entry, depth ) < 0 ) { return -1; } break; + case 15: case 30: if ( TableMessageRetainPayload( s, entry, depth ) < 0 ) { return -1; } break; + case 32: break; // nothing rides, and the outer length is what says so + case 12: case 31: case 33: + // a content copied whole carries no inner length: the outer one + // frames it, which is what a keyed slot's element takes + if ( !TableMessageRetainOpaque( s, entry, false ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +inline int64_t TableMessageRetainPayload( TableMessageRetainIn & s, const TableMessageEntry & entry, int32_t depth ) +{ + const int64_t began = s.out.out_at; + switch ( entry.kind ) + { + case 17: return -1; // THE NODE-INDEX CLASS, met inside a payload (§6.6) + case 0: return -1; // a KIND-0 ENTRY names no payload the file form has a kind for + case 32: TableRetainInLeb( s.out, 0 ); break; + case 30: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) + { + // A VARIANT REFERENCE OF ZERO IS THE ENUM'S None (§3): a value, + // not a reference, and it rides back as one + TableRetainInId( s.out, 0 ); + break; + } + uint64_t id = 0; + if ( !TableMessageRetainId( s, ref, id ) ) { return -1; } + TableRetainInId( s.out, id ); + break; + } + case 15: + { + uint64_t ref = 0; + if ( !s.r->get( ref, s.vocabulary->ref_bits ) ) { return -1; } + if ( ref == 0 ) { TableRetainInId( s.out, 0 ); break; } + TableMessageEntry arm; + if ( !TableMessageArmEntry( *s.vocabulary, ref, arm ) || TableRetainReservedId( arm.id ) ) { return -1; } + TableRetainInId( s.out, arm.id ); + TableRetainInRaw( s.out, &arm.kind, 1 ); + if ( TableMessageRetainFramed( s, arm, depth ) < 0 ) { return -1; } + break; + } + case 13: case 14: case 16: + if ( TableMessageRetainFramed( s, entry, depth ) < 0 ) { return -1; } + break; + case 12: case 31: case 33: + if ( !TableMessageRetainOpaque( s, entry, true ) ) { return -1; } + break; + default: + if ( !TableMessageRetainScalar( s, entry ) ) { return -1; } + break; + } + return s.out.out_at - began; +} + +// CAPTURE, the message form's load side. The entry has already been SKIPPED and +// counted unknown exactly as it always was, and the start offset is where its payload +// began: this re-reads those bits, and a record it cannot make changes nothing +// else about the read. +inline void TableMessageRetainCapture( TableRetain * retain, TableBitReader & r, const TableVocabulary & vocabulary, + int64_t index_bits, const TableMessageEntry & entry, + const TableRetainPath & path, TableReport * report, int64_t start ) +{ + if ( retain == NULL ) { return; } + if ( entry.kind == 17 ) + { + // A FIELD WHOSE PAYLOAD CARRIES A NODE INDEX: kind 17 itself, the + // excluded class at the outer kind (§6.6) + report->retain_lost++; + return; + } + const int64_t end = r.offset; + TableBitReader walk = r; + walk.offset = start; + + TableMessageRetainIn probe; + probe.out.in = NULL; + probe.out.size = 0; + probe.out.at = 0; + probe.out.ids = NULL; + probe.out.out = NULL; + probe.out.out_at = 0; + probe.r = &walk; + probe.vocabulary = &vocabulary; + probe.index_bits = index_bits; + const int64_t payload = TableMessageRetainPayload( probe, entry, 0 ); + if ( payload < 0 || walk.offset != end ) + { + // THE WALK CAN MEET WHAT THE SKIP STEPPED OVER, and its verdict changes + // nothing else: the record is dropped, one retain_lost counts, and + // malformed DOES NOT MOVE (§6.6). + report->retain_lost++; + return; + } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL (§6.6) + report->retain_lost++; + return; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, entry.id ); + record[24] = entry.kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableBitReader again = r; + again.offset = start; + TableMessageRetainIn write = probe; + write.r = &again; + write.out.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out.out_at = 0; + if ( TableMessageRetainPayload( write, entry, 0 ) < 0 ) { report->retain_lost++; return; } + retain->used += need; + retain->count++; + report->retained++; +} } // namespace streamdemo #endif // STREAMDEMO_SCHEMA_TABLE_RETAIN @@ -4685,14 +5117,17 @@ template inline bool TableNodeMessageSave( const Ctx & ctx, const inline int64_t HeaderMeasureBodyRetain( TableRetainIds & ids, const Header & value, TableRetain * retain, const TableRetainPath & path ); STREAMDEMO_TABLE_INLINE bool HeaderSaveBodyRetain( TableWriter & w, TableRetainIds & ids, const Header & value, TableRetain * retain, const TableRetainPath & path ); STREAMDEMO_TABLE_INLINE bool HeaderLoadBodyRetain( TableReader & r, Header & value, TableRetain * retain, const TableRetainPath & path ); +inline bool HeaderLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Header & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t ChunkMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Chunk & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ChunkSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chunk & value, TableRetain * retain, const TableRetainPath & path ); template inline bool ChunkSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Chunk & value, TableRetain * retain, const TableRetainPath & path ); inline bool ChunkLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Chunk & value, TableRetain * retain, const TableRetainPath & path ); +inline bool ChunkLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Chunk & value, TableRetain * retain, const TableRetainPath & path ); template inline int64_t FeedMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Feed & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FeedSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Feed & value, TableRetain * retain, const TableRetainPath & path ); template inline bool FeedSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Feed & value, TableRetain * retain, const TableRetainPath & path ); inline bool FeedLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Feed & value, TableRetain * retain, const TableRetainPath & path ); +inline bool FeedLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Feed & value, TableRetain * retain, const TableRetainPath & path ); inline int64_t HeaderMeasureBody( TableIds & ids, const Header & value ) { @@ -8658,6 +9093,73 @@ STREAMDEMO_TABLE_INLINE bool HeaderLoadBodyRetain( TableReader & r, Header & val } } +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool HeaderLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, int64_t index_bits, Header & value, TableRetain * retain, const TableRetainPath & path ) +{ + HeaderReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xc4bcadba8e631b86ull: // name + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 12 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 16 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.name, text, (size_t) kept ); + value.name[kept] = 0; + r.offset += (int64_t) n * 8; + value.name_length = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + template inline int64_t ChunkMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Chunk & value, TableRetain * retain, const TableRetainPath & path ) { @@ -8929,21 +9431,146 @@ inline bool ChunkLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Ch } } -template -inline int64_t FeedMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Feed & value, TableRetain * retain, const TableRetainPath & path ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool ChunkLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Chunk & value, TableRetain * retain, const TableRetainPath & path ) { - int64_t bytes = 1; // the ZERO REFERENCE that ends the body - if ( value.id != 0 ) { bytes += TableLebBytes( ids.ref( 0x08b72e07b55c3ac0ull ) ) + 1 + 4; } // id - if ( value.frame.type != FrameType::None ) // None elides — the absence of the field is the None + (void) nodes; (void) index_bits; + ChunkReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - bytes += TableLebBytes( ids.ref( 0xd8d4335628b35226ull ) ) + 1; - switch ( value.frame.type ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - case FrameType::None: break; - case FrameType::Header: + case 0x855b556730a34a05ull: // data { - int64_t arm_payload = 0; - const uint64_t arm_ref = ids.ref( 0x3a506cb501761960ull ); + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 6 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 8 ) { kept = 8; report->clamped++; } else { kept = (int32_t) n; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !r.get( by, 8 ) ) { report->malformed = true; return false; } + if ( (int32_t) i < kept ) { value.data[i] = (uint8_t) by; } + } + value.data_length = kept; + } + break; + } + case 0xe5316cbaa025f028ull: // next + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 17 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t node_index = 0; + if ( !r.get( node_index, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.next, node_index, 0xcf4368dcf951e082ull, report ); // *Chunk + } + break; + } + case 0x5cc201a9f1b8274eull: // links + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.links[i] : scratch ), node_index_2, 0xcf4368dcf951e082ull, report ); // *Chunk + } + } + value.links_count = kept; + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +template +inline int64_t FeedMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Feed & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.id != 0 ) { bytes += TableLebBytes( ids.ref( 0x08b72e07b55c3ac0ull ) ) + 1 + 4; } // id + if ( value.frame.type != FrameType::None ) // None elides — the absence of the field is the None + { + bytes += TableLebBytes( ids.ref( 0xd8d4335628b35226ull ) ) + 1; + switch ( value.frame.type ) + { + case FrameType::None: break; + case FrameType::Header: + { + int64_t arm_payload = 0; + const uint64_t arm_ref = ids.ref( 0x3a506cb501761960ull ); { const int64_t arm_body = HeaderMeasureBodyRetain( ids, value.frame.header, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ); if ( arm_body < 0 ) { return -1; } @@ -9456,140 +10083,405 @@ inline bool FeedLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Fee } } -// ChunkNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it -// already owns. -// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the -// index is the region directory's, which Load fills from the wire's framing -// and nothing afterwards renumbers. -inline void ChunkNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) -{ - switch ( type_id ) - { - case 0xcf4368dcf951e082ull: ChunkLoadBodyRetain( r, nodes, *(Chunk *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Chunk - default: break; - } -} - -// ChunkLoadRetain: decode the tolerant wire into the caller's exact-sized region and -// return the root. LOAD IS A SCAN, and that is the whole of its bound: it -// follows no reference, so there is no depth cap, no visited set and no -// ordering rule on the indices. Partial results are kept, as everywhere on -// this wire — the report says what happened. NULL means the CALLER's buffer -// was wrong. -// UNDER RETENTION it also fills the caller's two stores with the fields -// this build cannot name, and the report carries what it could not keep -// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the -// reader's data is exactly what it would have been with retention off. -inline const Chunk * ChunkLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +// UNDER RETENTION the same body keeps the fields this build cannot name, +// resolved against the CONNECTION'S VOCABULARY instead of a trailer, with the +// path threaded at every child-body descent (docs/SPEC-TABLES.md §3.3, §6.6). +// The reader's own data is exactly what it would have been with retention off. +inline bool FeedLoadMessageBodyRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Feed & value, TableRetain * retain, const TableRetainPath & path ) { - TableReport ignored; - TableReport * out = report != NULL ? report : &ignored; - // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: - // a file that is both a newer form and damaged is a REFUSAL and never - // damage (docs/SPEC-TABLES.md §3). - TableIdTable ids_table; - int64_t body_bytes = 0; - const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); - if ( verdict != TableOpenOk ) - { - if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } - return NULL; - } - if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) - { - out->malformed = true; // a byte no field claims, before the table (§3) - return NULL; - } - const uint8_t * const wire = wire_file + 1; - const int64_t wire_bytes = body_bytes; - if ( region == NULL || region_bytes < (int64_t) sizeof( Chunk ) ) { out->malformed = true; return NULL; } - if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } - memset( region, 0, (size_t) region_bytes ); - uint64_t type_id = 0; - const uint8_t * body = NULL; - int64_t length = 0; - - // the record count and the data bytes, from the FRAMING alone - TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed - int64_t data = TableAlignUp64( (int64_t) sizeof( Chunk ) ); - int64_t records = 0; - { - TableReport counting; - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); - while ( TableNodeScanNext( scan, type_id, body, length ) ) - { - records++; - int64_t storage = ChunkNodeStorage( type_id, length, reason ); - if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } - if ( storage > 0 ) { data += storage; } - } - } - int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); - if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } - - TableNodeMap nodes; - nodes.base = region; - nodes.entries = (const TableNodeDirEntry *) ( region + data ); - nodes.count = records + 1; - TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); - directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) - directory[0].type_id = 0xcf4368dcf951e082ull; - Chunk * root = new ( region ) Chunk; // lifetime only: LoadBody's first act is ChunkReset - ChunkReset( *root ); - - // LoadRetain RESETS BOTH STORES and writes into neither id list: a - // retained record carries its field's identity in the record itself, - // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer - // belongs to this region from here on. - TableRetainReset( retain, nodes, region ); - - // PASS ONE: fill the numbering from the framing, so that an index - // resolves whichever way it points. It reads no body. + (void) nodes; (void) index_bits; + FeedReset( value ); + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so whatever + // an earlier occurrence of it left is discarded before the winning one is + // read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t used = TableAlignUp64( (int64_t) sizeof( Chunk ) ); - int64_t k = 0; - int32_t unknown_records = 0; // counted once the scan is known whole - while ( TableNodeScanNext( scan, type_id, body, length ) ) + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) { - int64_t storage = ChunkNodeStorage( type_id, length, reason ); - if ( storage <= 0 ) - { - // a record whose type id this build cannot name KEEPS ITS - // INDEX, is counted once here and not once per pointer, and - // every reference to it reads null (§3.1) - unknown_records++; - directory[k + 1].offset = kTableNodeAbsent; - directory[k + 1].type_id = type_id; - } - else + case 0x08b72e07b55c3ac0ull: // id { - directory[k + 1].offset = (uint64_t) used; - directory[k + 1].type_id = type_id; - ChunkNodePlace( type_id, region + used, length ); - used += storage; - } - k++; - } - nodes.good = TableNodeScanWhole( scan ); - // the table is whole or it is nothing: a scan that failed counts - // malformed and NOT the unknowns it met on the way, because the - // numbering they belonged to does not exist (§3.1) - // A NODE RECORD whose type id this reader cannot name is one of the - // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one - // back means renumbering a graph the writer numbers from its own edges. - if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } - } - - // PASS TWO: decode each body into its own storage. A forward index - // resolves without scratch, because pass one already placed every node. - if ( nodes.good ) - { - TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); - int64_t k = 0; - while ( TableNodeScanNext( scan, type_id, body, length ) ) - { - if ( directory[k + 1].offset != kTableNodeAbsent ) + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.id = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.id = decoded_v; + } + break; + } + case 0xd8d4335628b35226ull: // frame + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 15 || entry.elem_kind != 0 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + TableRetainDiscardField( retain, path, 1 ); + { + uint64_t arm_ref = 0; + if ( !r.get( arm_ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + TableMessageEntry arm; + if ( arm_ref == 0 ) { value.frame.type = FrameType::None; } + else if ( !TableMessageArmEntry( vocabulary, arm_ref, arm ) ) { report->malformed = true; return false; } + else + { + switch ( arm.id ) + { + case 0x3a506cb501761960ull: // header + { + if ( arm.kind != 13 || arm.elem_kind != 0 ) + { + value.frame.type = FrameType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.frame.type = FrameType::Header; + if ( !HeaderLoadMessageBodyRetain( r, vocabulary, report, index_bits, value.frame.header, retain, TableRetainStepInto( path, 1, (uint32_t) ( 0 ) ) ) ) { return false; } + break; + } + case 0x0f838176873c8e22ull: // chunk + { + if ( arm.kind != 13 || arm.elem_kind != 0 ) + { + value.frame.type = FrameType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.frame.type = FrameType::Chunk; + if ( !ChunkLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, value.frame.chunk, retain, TableRetainStepInto( path, 1, (uint32_t) ( 1 ) ) ) ) { return false; } + break; + } + case 0xbf4b9bad694f4809ull: // link + { + if ( arm.kind != 17 || arm.elem_kind != 0 ) + { + value.frame.type = FrameType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.frame.type = FrameType::Link; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, value.frame.link, node_index_2, 0xcf4368dcf951e082ull, report ); // *Chunk + } + break; + } + case 0x56d7ab194448a4f3ull: // tag + { + if ( arm.kind != 8 || arm.elem_kind != 0 ) + { + if ( TableKindWidens( arm.kind, 8 ) ) + { + value.frame.type = FrameType::Tag; + memset( (void *) &value.frame.tag, 0, sizeof( value.frame.tag ) ); // selection establishes the arm (§2.6) + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + if ( (uint64_t) decoded_wide_2 > 4294967295ull ) { decoded_wide_2 = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v_2 = (uint32_t) decoded_wide_2; + value.frame.tag = decoded_v_2; + } + report->widened++; + break; + } + value.frame.type = FrameType::None; report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + value.frame.type = FrameType::Tag; + { + const int64_t width_2 = arm.value_bits; + uint64_t raw_2 = 0; + if ( width_2 < 0 || !r.get( raw_2, width_2 ) ) { report->malformed = true; return false; } + int64_t decoded_wide_2 = (int64_t) raw_2; + if ( arm.packing == 1 ) { decoded_wide_2 = (int64_t) ( raw_2 + (uint64_t) arm.base_lo ); } + if ( (uint64_t) decoded_wide_2 > 4294967295ull ) { decoded_wide_2 = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v_2 = (uint32_t) decoded_wide_2; + value.frame.tag = decoded_v_2; + } + break; + } + default: + value.frame.type = FrameType::None; report->unknown++; report->retain_lost++; + if ( !TableMessageSkip( r, vocabulary, index_bits, arm ) ) { report->malformed = true; return false; } + break; + } + } + } + break; + } + case 0x0c519da7a1f958c5ull: // parts + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 4 ) { kept = 4; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.parts[i] : scratch ), node_index_2, 0xcf4368dcf951e082ull, report ); // *Chunk + } + } + value.parts_count = kept; + } + break; + } + case 0x0369250deb889a31ull: // pair + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.pair[i] : scratch ), node_index_2, 0xcf4368dcf951e082ull, report ); // *Chunk + } + } + } + break; + } + default: + report->unknown++; + { + const int64_t unknown_at = r.offset; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at ); + } + break; + } + } +} + +// ChunkNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void ChunkNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + switch ( type_id ) + { + case 0xcf4368dcf951e082ull: ChunkLoadBodyRetain( r, nodes, *(Chunk *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Chunk + default: break; + } +} + +// ChunkLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Chunk * ChunkLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Chunk ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t data = TableAlignUp64( (int64_t) sizeof( Chunk ) ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = ChunkNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xcf4368dcf951e082ull; + Chunk * root = new ( region ) Chunk; // lifetime only: LoadBody's first act is ChunkReset + ChunkReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( (int64_t) sizeof( Chunk ) ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = ChunkNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + ChunkNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) { TableReader sub( body, length, out, &ids_table ); ChunkNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); @@ -9606,6 +10498,160 @@ inline const Chunk * ChunkLoadRetain( uint8_t * region, int64_t region_bytes, co return root; } +// ChunkNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool ChunkNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0xcf4368dcf951e082ull: ok = ChunkLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(Chunk *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Chunk + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// ChunkLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool ChunkLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Chunk * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !ChunkMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = ChunkNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + ChunkNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Chunk ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xcf4368dcf951e082ull; + Chunk * root = new ( region + used ) Chunk; // lifetime only: LoadMessageBody's first act is ChunkReset + ChunkReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !ChunkNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return ChunkLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// ChunkLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool ChunkLoadRetainMessages( const Chunk ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !ChunkLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // ChunkMeasureRetain and ChunkSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save @@ -9874,6 +10920,160 @@ inline const Feed * FeedLoadRetain( uint8_t * region, int64_t region_bytes, cons return root; } +// FeedNodeMessageBodyRetain: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is this body's own directory position, which the batch's load fills +// from the framing and nothing afterwards renumbers. +inline bool FeedNodeMessageBodyRetain( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + bool ok = false; + switch ( type_id ) + { + case 0xcf4368dcf951e082ull: ok = ChunkLoadMessageBodyRetain( r, vocabulary, report, nodes, index_bits, *(Chunk *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Chunk + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + return ok; +} + +// FeedLoadMessageBodyIntoRetain: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +// UNDER RETENTION the body's own retention buffer is reset here and belongs +// to it from here on: a batch takes ONE REGION and ONE BUFFER A BODY, because +// each body carries its own node directory inside that one region and a +// record's first step is an index into it (docs/SPEC-TABLES.md §3.3, §6.6). +inline bool FeedLoadMessageBodyIntoRetain( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Feed * & root_out, TableRetain * retain ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !FeedMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = FeedNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + FeedNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Feed ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x3a8fe3852a245245ull; + Feed * root = new ( region + used ) Feed; // lifetime only: LoadMessageBody's first act is FeedReset + FeedReset( *root ); + root_out = root; + used += root_bytes; + nodes.good = true; + // A NODE RECORD whose type id this reader cannot name is one of the SIX + // EXCLUDED CLASSES (§6.6): it is a whole node, and putting one back means + // renumbering a graph the writer numbers from its own edges. + out->unknown += unknown_records; + out->retain_lost += unknown_records; + // THE BUFFER BELONGS TO THIS BODY from here on: it resets both stores and + // writes into neither id list, because a retained record carries its + // field's identity in the record itself (§6.6). + TableRetainReset( retain, nodes, region ); + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !FeedNodeMessageBodyRetain( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + return FeedLoadMessageBodyRetain( r, vocabulary, out, nodes, index_bits, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); +} + +// FeedLoadRetainMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +// A BATCH TAKES ONE REGION AND ONE RETENTION BUFFER A BODY (§3.3, §6.6): +// `retains` is an array parallel to `roots`, each entry reset by this call +// and each holding the records of the body it belongs to, which is what keeps +// a record's first step an index into that body's own node directory. A +// SaveRetain from `roots[k]` takes `retains[k]` and is the file form's own +// pair unchanged: retention writing FORM 2 refuses by name (§3.3). +inline bool FeedLoadRetainMessages( const Feed ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableRetain * retains, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !FeedLoadMessageBodyIntoRetain( br.r, vocabulary, out, region, region_bytes, used, roots[b], retains != NULL ? &retains[b] : NULL ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + // FeedMeasureRetain and FeedSaveRetain: the pair, with the retained tail in // every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same // records under the same walk, so Measure's answer is the size the save diff --git a/testdata/wire/tables/retain_conn.bin b/testdata/wire/tables/retain_conn.bin new file mode 100644 index 000000000..9132d362f Binary files /dev/null and b/testdata/wire/tables/retain_conn.bin differ diff --git a/testdata/wire/tables/retain_message.bin b/testdata/wire/tables/retain_message.bin new file mode 100644 index 000000000..993c1753c Binary files /dev/null and b/testdata/wire/tables/retain_message.bin differ diff --git a/testdata/wire/tables/retain_message_save_0.bin b/testdata/wire/tables/retain_message_save_0.bin new file mode 100644 index 000000000..62b372e13 Binary files /dev/null and b/testdata/wire/tables/retain_message_save_0.bin differ diff --git a/testdata/wire/tables/retain_message_save_1.bin b/testdata/wire/tables/retain_message_save_1.bin new file mode 100644 index 000000000..2485f8fe5 Binary files /dev/null and b/testdata/wire/tables/retain_message_save_1.bin differ diff --git a/tools/sabotage/message.go b/tools/sabotage/message.go index f27c26950..32c7dbe76 100644 --- a/tools/sabotage/message.go +++ b/tools/sabotage/message.go @@ -111,8 +111,8 @@ var messageSabotages = map[string][]edit{ // AN OVER-LONG ARRAY CLAMPS BY WALKING THE SURPLUS: stop at the bound // instead and the next field lands on the wrong bit. "message-clamp-drops-surplus": {{ - old: "\tfor i := uint64(0); i < walk; i++ {\n\t\tvar sink tabletext.Cell\n\t\tcell := &sink\n\t\tif f.Array == ir.ArrayList {\n", - new: "\tfor i := uint64(0); i < kept; i++ { // SABOTAGED: the surplus is not walked\n\t\tvar sink tabletext.Cell\n\t\tcell := &sink\n\t\tif f.Array == ir.ArrayList {\n", + old: "\tfor i := uint64(0); i < walk; i++ {\n\t\tvar sink tabletext.Cell\n\t\tcell := &sink\n\t\tmine := true\n", + new: "\tfor i := uint64(0); i < kept; i++ { // SABOTAGED: the surplus is not walked\n\t\tvar sink tabletext.Cell\n\t\tcell := &sink\n\t\tmine := true\n", }}, // A SKIPPED STRING ALIGNS BEFORE ITS BYTES exactly as a read one does. @@ -273,8 +273,8 @@ var messageRoundTwoSabotages = map[string][]edit{ // A DISCARDED SURPLUS ELEMENT NEVER ACQUIRES A LIVE DESTINATION (M1): land // it on element zero instead. "message-surplus-lands-on-zero": {{ - old: "\t\tvar sink tabletext.Cell\n\t\tcell := &sink\n\t\tif f.Array == ir.ArrayList {\n", - new: "\t\tcell := &fv.Elems[0] // SABOTAGED: a surplus element overwrites element zero\n\t\tif f.Array == ir.ArrayList {\n", + old: "\t\tvar sink tabletext.Cell\n\t\tcell := &sink\n\t\tmine := true\n\t\tswitch {\n\t\tcase f.Array == ir.ArrayList:\n", + new: "\t\tcell := &fv.Elems[0] // SABOTAGED: a surplus element overwrites element zero\n\t\tmine := true\n\t\tswitch {\n\t\tcase f.Array == ir.ArrayList:\n", }}, // A RANGED 128-BIT VALUE READS AT ITS ANNOUNCED WIDTH (M2): read the raw @@ -426,3 +426,18 @@ var messageTextSabotages = map[string][]edit{ new: "\t\tg.pf(\" // SABOTAGED: the blob content rule is gone\\n\")\n", }}, } + +// THE MESSAGE FORM'S RETENTION (docs/SPEC-TABLES.md §3.3, §6.6). The unknown +// arm of a retaining message body SKIPS the entry and then re-reads the bits +// it delimited, which is the whole of the form-2 capture. Take the second half +// away and the skip is all that is left: the read is unchanged to the byte, +// every counter but the two retention ones stands, and nothing about the batch +// says a field was lost. What goes red is the pinned batch's own row. +var messageRetainSabotages = map[string][]edit{ + "message-retain-no-capture": {{ + old: "\t\tg.pf(\" TableMessageRetainCapture( retain, r, vocabulary, index_bits, entry, path, report, unknown_at );\\n\")\n", + new: "\t\tg.pf(\" (void) unknown_at; // SABOTAGED: the message path never enters the resolving walk\\n\")\n", + }}, +} + +func init() { maps.Copy(sabotages, messageRetainSabotages) }